1# -*-perl-*-
2use strict;
3use warnings;
4use Test::More;
5use FindBin qw($Bin);
6use MogileFS::Server;
7use MogileFS::Test;
8use HTTP::Request;
9find_mogclient_or_skip();
10
11my $sto = eval { temp_store(); };
12if (!$sto) {
13    plan skip_all => "Can't create temporary test database: $@";
14    exit 0;
15}
16
17use File::Temp;
18my %mogroot;
19$mogroot{1} = File::Temp::tempdir( CLEANUP => 1 );
20$mogroot{2} = File::Temp::tempdir( CLEANUP => 1 );
21my $dev2host = { 1 => 1, 2 => 2, };
22foreach (sort { $a <=> $b } keys %$dev2host) {
23    my $root = $mogroot{$dev2host->{$_}};
24    mkdir("$root/dev$_") or die "Failed to create dev$_ dir: $!";
25}
26
27my $ms1 = create_mogstored("127.0.1.1", $mogroot{1});
28ok($ms1, "got mogstored1");
29my $ms2 = create_mogstored("127.0.1.2", $mogroot{2});
30ok($ms1, "got mogstored2");
31
32try_for(30, sub {
33    print "Waiting on usage...\n";
34    -e "$mogroot{1}/dev1/usage" && -e "$mogroot{2}/dev2/usage";
35});
36
37my $tmptrack = create_temp_tracker($sto);
38ok($tmptrack);
39
40my $admin = IO::Socket::INET->new(PeerAddr => '127.0.0.1:7001');
41$admin or die "failed to create admin socket: $!";
42my $moga = MogileFS::Admin->new(hosts => [ "127.0.0.1:7001" ]);
43my $mogc = MogileFS::Client->new(
44                                 domain => "testdom",
45                                 hosts  => [ "127.0.0.1:7001" ],
46                                 );
47my $be = $mogc->{backend}; # gross, reaching inside of MogileFS::Client
48
49# test some basic commands to backend
50ok($tmptrack->mogadm("domain", "add", "testdom"), "created test domain");
51ok($tmptrack->mogadm("class", "add", "testdom", "2copies", "--mindevcount=2"), "created 2copies class in testdom");
52ok($tmptrack->mogadm("class", "add", "testdom", "1copy", "--mindevcount=1"), "created 1copy class in testdom");
53
54ok($tmptrack->mogadm("host", "add", "hostA", "--ip=127.0.1.1", "--status=alive"), "created hostA");
55ok($tmptrack->mogadm("host", "add", "hostB", "--ip=127.0.1.2", "--status=alive"), "created hostB");
56
57ok($tmptrack->mogadm("device", "add", "hostA", 1), "created dev1 on hostA");
58ok($tmptrack->mogadm("device", "add", "hostB", 2), "created dev2 on hostB");
59
60sub wait_for_monitor {
61    my $be = shift;
62    my $was = $be->{timeout};  # can't use local on phash :(
63    $be->{timeout} = 10;
64    ok($be->do_request("clear_cache", {}), "waited for monitor")
65        or die "Failed to wait for monitor";
66    ok($be->do_request("clear_cache", {}), "waited for monitor")
67        or die "Failed to wait for monitor";
68    $be->{timeout} = $was;
69}
70
71sub full_fsck {
72    my $tmptrack = shift;
73
74    ok($tmptrack->mogadm("fsck", "stop"), "stop fsck");
75    ok($tmptrack->mogadm("fsck", "clearlog"), "clear fsck log");
76    ok($tmptrack->mogadm("fsck", "reset"), "reset fsck");
77    ok($tmptrack->mogadm("fsck", "start"), "started fsck");
78}
79
80wait_for_monitor($be);
81
82my ($req, $rv, %opts, @paths, @fsck_log);
83my $ua = LWP::UserAgent->new;
84
85use Data::Dumper;
86use Digest::MD5 qw/md5_hex/;
87
88# verify upload checksum
89{
90    my $key = "ok";
91    %opts = ( domain => "testdom", class => "1copy", key => $key );
92    $rv = $be->do_request("create_open", \%opts);
93    %opts = %$rv;
94    ok($rv && $rv->{path}, "create_open succeeded");
95    $req = HTTP::Request->new(PUT => $rv->{path});
96    $req->content("blah");
97    $rv = $ua->request($req);
98    ok($rv->is_success, "PUT successful");
99    $opts{key} = $key;
100    $opts{domain} = "testdom";
101    $opts{checksum} = "MD5:".md5_hex('blah');
102    $opts{checksumverify} = 1;
103    $rv = $be->do_request("create_close", \%opts);
104    ok($rv, "checksum verified successfully");
105    is($sto->get_checksum($opts{fid}), undef, "checksum not saved");
106    ok($mogc->file_info($key), "file_info($key) is sane");
107}
108
109# corrupted upload checksum fails
110{
111    my $key = 'corrupt';
112    %opts = ( domain => "testdom", class => "1copy", key => $key );
113    $rv = $be->do_request("create_open", \%opts);
114    %opts = %$rv;
115    ok($rv && $rv->{path}, "create_open succeeded");
116    $req = HTTP::Request->new(PUT => $rv->{path});
117    $req->content("blah");
118    $rv = $ua->request($req);
119    ok($rv->is_success, "PUT successful");
120    $opts{key} = $key;
121    $opts{domain} = "testdom";
122
123    $opts{checksumverify} = 1;
124    $opts{checksum} = "MD5:".md5_hex('fail');
125    $rv = $be->do_request("create_close", \%opts);
126    ok(!defined($rv), "checksum verify noticed mismatch");
127    my $hex = md5_hex('blah');
128    is('checksum_mismatch', $be->{lasterr}, "error code is correct");
129    ok($be->{lasterrstr} =~ /actual: MD5:$hex;/, "error message shows actual:");
130    is($sto->get_checksum($opts{fid}), undef, "checksum not saved");
131    is($mogc->file_info($key), undef, "$key not uploaded");
132}
133
134# enable saving MD5 checksums in "2copies" class
135{
136    %opts = ( domain => "testdom", class => "2copies",
137              hashtype => "MD5", mindevcount => 2 );
138    ok($be->do_request("update_class", \%opts), "update class");
139    wait_for_monitor($be);
140}
141
142# save new row to checksum table
143{
144    my $key = 'savecksum';
145
146    want($admin, 0, "replicate");
147
148    %opts = ( domain => "testdom", class => "2copies", key => $key );
149    $rv = $be->do_request("create_open", \%opts);
150    %opts = %$rv;
151    ok($rv && $rv->{path}, "create_open succeeded");
152    $req = HTTP::Request->new(PUT => $rv->{path});
153    $req->content("blah");
154    $rv = $ua->request($req);
155    ok($rv->is_success, "PUT successful");
156    $opts{key} = $key;
157    $opts{domain} = "testdom";
158    $opts{checksum} = "MD5:".md5_hex('blah');
159    $opts{checksumverify} = 1;
160    $rv = $be->do_request("create_close", \%opts);
161    ok($rv, "checksum verified successfully");
162    my $row = $sto->get_checksum($opts{fid});
163    ok($row, "checksum saved");
164    my $info = $mogc->file_info($key);
165    ok($info, "file_info($key) is sane");
166    is($info->{checksum}, "MD5:".md5_hex('blah'), 'checksum shows up');
167    is($sto->delete_checksum($info->{fid}), 1, "$key checksum row deleted");
168    $info = $mogc->file_info($key);
169    is($info->{checksum}, "MISSING", 'checksum is MISSING after delete');
170
171    want($admin, 1, "replicate");
172
173    # wait for replicate to recreate checksum
174    try_for(30, sub {
175        @paths = $mogc->get_paths($key);
176        scalar(@paths) != 1;
177    });
178    is(scalar(@paths), 2, "replicate successfully with good checksum");
179
180    $info = $mogc->file_info($key);
181    is($info->{checksum}, "MD5:".md5_hex('blah'), 'checksum recreated on repl');
182}
183
184# flip checksum classes around
185{
186    my @classes;
187    %opts = ( domain => "testdom", class => "1copy", mindevcount => 1 );
188
189    $opts{hashtype} = "NONE";
190    ok($be->do_request("update_class", \%opts), "update class");
191    @classes = grep { $_->{classname} eq '1copy' } $sto->get_all_classes;
192    is($classes[0]->{hashtype}, undef, "hashtype unset");
193
194    $opts{hashtype} = "MD5";
195    ok($be->do_request("update_class", \%opts), "update class");
196    @classes = grep { $_->{classname} eq '1copy' } $sto->get_all_classes;
197    is($classes[0]->{hashtype}, 1, "hashtype is 1 (MD5)");
198
199    $opts{hashtype} = "NONE";
200    ok($be->do_request("update_class", \%opts), "update class");
201    @classes = grep { $_->{classname} eq '1copy' } $sto->get_all_classes;
202    is($classes[0]->{hashtype}, undef, "hashtype unset");
203}
204
205# save checksum on replicate, client didn't care to provide one
206{
207    my $key = 'lazycksum';
208
209    want($admin, 0, "replicate");
210
211    my $fh = $mogc->new_file($key, "2copies");
212    print $fh "lazy";
213    ok(close($fh), "closed file");
214    my $info = $mogc->file_info($key);
215    is($info->{checksum}, 'MISSING', 'checksum is MISSING');
216
217    want($admin, 1, "replicate");
218
219    try_for(30, sub {
220        @paths = $mogc->get_paths($key);
221        scalar(@paths) != 1;
222    });
223    is(scalar(@paths), 2, "replicate successfully with good checksum");
224
225    $info = $mogc->file_info($key);
226    is($info->{checksum}, "MD5:".md5_hex("lazy"), 'checksum is set after repl');
227}
228
229# fsck recreates checksum when missing
230{
231    my $key = 'lazycksum';
232    my $info = $mogc->file_info($key);
233    $sto->delete_checksum($info->{fid});
234    $info = $mogc->file_info($key);
235    is($info->{checksum}, "MISSING", "checksum is missing");
236    full_fsck($tmptrack);
237
238    try_for(30, sub {
239        $info = $mogc->file_info($key);
240        $info->{checksum} ne "MISSING";
241    });
242    is($info->{checksum}, "MD5:".md5_hex("lazy"), 'checksum is set after fsck');
243
244    @fsck_log = $sto->fsck_log_rows;
245    is(scalar(@fsck_log), 1, "fsck log has one row");
246    is($fsck_log[0]->{fid}, $info->{fid}, "fid matches in fsck log");
247    is($fsck_log[0]->{evcode}, "NSUM", "missing checksum logged");
248}
249
250# fsck fixes a file corrupt file
251{
252    my $key = 'lazycksum';
253    my $info = $mogc->file_info($key);
254    @paths = $mogc->get_paths($key);
255    is(scalar(@paths), 2, "2 paths for lazycksum");
256    $req = HTTP::Request->new(PUT => $paths[0]);
257    $req->content("LAZY");
258    $rv = $ua->request($req);
259    ok($rv->is_success, "upload to corrupt a file successful");
260    is($ua->get($paths[0])->content, "LAZY", "file successfully corrupted");
261    is($ua->get($paths[1])->content, "lazy", "paths[1] not corrupted");
262
263    full_fsck($tmptrack);
264
265    try_for(30, sub {
266        @fsck_log = $sto->fsck_log_rows;
267        scalar(@fsck_log) != 0;
268    });
269
270    is(scalar(@fsck_log), 1, "fsck log has one row");
271    is($fsck_log[0]->{fid}, $info->{fid}, "fid matches in fsck log");
272    is($fsck_log[0]->{evcode}, "REPL", "repl for mismatched checksum logged");
273
274    try_for(30, sub {
275        @paths = $mogc->get_paths($key);
276        scalar(@paths) >= 2;
277    });
278
279    is(scalar(@paths), 2, "2 paths for key after replication");
280    is($ua->get($paths[0])->content, "lazy", "paths[0] is correct");
281    is($ua->get($paths[1])->content, "lazy", "paths[1] is correct");
282    $info = $mogc->file_info($key);
283    is($info->{checksum}, "MD5:".md5_hex("lazy"), 'checksum unchanged after fsck');
284}
285
286# fsck notices when all files are corrupt
287{
288    my $key = 'lazycksum';
289    my $info = $mogc->file_info($key);
290    @paths = $mogc->get_paths($key);
291    is(scalar(@paths), 2, "2 paths for lazycksum");
292
293    $req = HTTP::Request->new(PUT => $paths[0]);
294    $req->content("0000");
295    $rv = $ua->request($req);
296    ok($rv->is_success, "upload to corrupt a file successful");
297    is($ua->get($paths[0])->content, "0000", "successfully corrupted");
298
299    $req = HTTP::Request->new(PUT => $paths[1]);
300    $req->content("1111");
301    $rv = $ua->request($req);
302    ok($rv->is_success, "upload to corrupt a file successful");
303    is($ua->get($paths[1])->content, "1111", "successfully corrupted");
304
305    full_fsck($tmptrack);
306
307    try_for(30, sub {
308        @fsck_log = $sto->fsck_log_rows;
309        scalar(@fsck_log) != 0;
310    });
311
312    is(scalar(@fsck_log), 1, "fsck log has one row");
313    is($fsck_log[0]->{fid}, $info->{fid}, "fid matches in fsck log");
314    is($fsck_log[0]->{evcode}, "BSUM", "BSUM logged");
315
316    @paths = $mogc->get_paths($key);
317    is(scalar(@paths), 2, "2 paths for checksum");
318    @paths = sort( map { $ua->get($_)->content } @paths);
319    is(join(', ', @paths), "0000, 1111", "corrupted content preserved");
320}
321
322# reuploaded checksum row clobbers old checksum
323{
324    my $key = 'lazycksum';
325    my $info = $mogc->file_info($key);
326
327    ok($sto->get_checksum($info->{fid}), "old checksum row exists");
328
329    my $fh = $mogc->new_file($key, "2copies");
330    print $fh "HAZY";
331    ok(close($fh), "closed replacement file (lazycksum => HAZY)");
332
333    try_for(30, sub { ! $sto->get_checksum($info->{fid}); });
334    is($sto->get_checksum($info->{fid}), undef, "old checksum is gone");
335}
336
337# completely corrupted files with no checksum row
338{
339    my $key = 'lazycksum';
340    try_for(30, sub {
341        @paths = $mogc->get_paths($key);
342        scalar(@paths) >= 2;
343    });
344    is(scalar(@paths), 2, "replicated succesfully");
345
346    my $info = $mogc->file_info($key);
347    is($info->{checksum}, "MD5:".md5_hex("HAZY"), "checksum created on repl");
348
349    $req = HTTP::Request->new(PUT => $paths[0]);
350    $req->content("MAYB");
351    $rv = $ua->request($req);
352    ok($rv->is_success, "upload to corrupt a file successful");
353    is($ua->get($paths[0])->content, "MAYB", "successfully corrupted (MAYB)");
354
355    $req = HTTP::Request->new(PUT => $paths[1]);
356    $req->content("CRZY");
357    $rv = $ua->request($req);
358    ok($rv->is_success, "upload to corrupt a file successful");
359    is($ua->get($paths[1])->content, "CRZY", "successfully corrupted (CRZY)");
360
361    is($sto->delete_checksum($info->{fid}), 1, "nuke new checksum");
362    $info = $mogc->file_info($key);
363    is($info->{checksum}, "MISSING", "checksum is MISSING");
364
365    full_fsck($tmptrack);
366
367    try_for(30, sub {
368        @fsck_log = $sto->fsck_log_rows;
369        scalar(@fsck_log) != 0;
370    });
371
372    is(scalar(@fsck_log), 1, "fsck log has one row");
373    is($fsck_log[0]->{fid}, $info->{fid}, "fid matches in fsck log");
374    is($fsck_log[0]->{evcode}, "BSUM", "BSUM logged");
375}
376
377# disable MD5 checksums in "2copies" class
378{
379    %opts = ( domain => "testdom", class => "2copies",
380              hashtype => "NONE", mindevcount => 2 );
381    ok($be->do_request("update_class", \%opts), "update class");
382    wait_for_monitor($be);
383}
384
385# use fsck_checksum=MD5 instead of per-class checksums
386{
387    my $key = 'lazycksum';
388    my $info = $mogc->file_info($key);
389    $sto->delete_checksum($info->{fid});
390
391    ok($tmptrack->mogadm("settings", "set", "fsck_checksum", "MD5"), "enable fsck_checksum=MD5");
392    wait_for_monitor($be);
393    full_fsck($tmptrack);
394
395    try_for(30, sub {
396        @fsck_log = $sto->fsck_log_rows;
397        scalar(@fsck_log) != 0;
398    });
399    is(scalar(@fsck_log), 1, "fsck log has one row");
400    is($fsck_log[0]->{fid}, $info->{fid}, "fid matches in fsck log");
401    is($fsck_log[0]->{evcode}, "MSUM", "MSUM logged");
402}
403
404# ensure server setting is visible
405use MogileFS::Admin;
406{
407    my $settings = $moga->server_settings;
408    is($settings->{fsck_checksum}, 'MD5', "fsck_checksum server setting visible");
409}
410
411use MogileFS::Config;
412
413# disable checksumming entirely, regardless of class setting
414{
415    %opts = ( domain => "testdom", class => "2copies",
416              hashtype => "MD5", mindevcount => 2 );
417    ok($be->do_request("update_class", \%opts), "update class");
418    wait_for_monitor($be);
419
420    ok($tmptrack->mogadm("settings", "set", "fsck_checksum", "off"), "set fsck_checksum=off");
421    wait_for_monitor($be);
422    my $settings = $moga->server_settings;
423    is($settings->{fsck_checksum}, 'off', "fsck_checksum server setting visible");
424    full_fsck($tmptrack);
425    my $nr;
426    try_for(1000, sub {
427        $nr = $sto->file_queue_length(FSCK_QUEUE);
428        $nr eq '0';
429    });
430    is($nr, '0', "fsck finished");
431    @fsck_log = $sto->fsck_log_rows;
432    is(scalar(@fsck_log), 0, "fsck log is empty with fsck_checksum=off");
433}
434
435# set fsck_checksum=class and ensure that works again
436{
437    my $info = $mogc->file_info('lazycksum');
438    ok($tmptrack->mogadm("settings", "set", "fsck_checksum", "class"), "set fsck_checksum=class");
439    wait_for_monitor($be);
440    my $settings = $moga->server_settings;
441    ok(! defined($settings->{fsck_checksum}), "fsck_checksum=class server setting hidden (default)");
442    full_fsck($tmptrack);
443
444    try_for(30, sub {
445        @fsck_log = $sto->fsck_log_rows;
446        scalar(@fsck_log) != 0;
447    });
448
449    is(scalar(@fsck_log), 1, "fsck log has one row");
450    is($fsck_log[0]->{fid}, $info->{fid}, "fid matches in fsck log");
451    is($fsck_log[0]->{evcode}, "BSUM", "BSUM logged");
452}
453
454done_testing();
455