1# -*-perl-*- 2use strict; 3use warnings; 4use Test::More; 5use FindBin qw($Bin); 6use MogileFS::Server; 7use MogileFS::Test; 8use HTTP::Request; 9find_mogclient_or_skip(); 10 11my $sto = eval { temp_store(); }; 12if (!$sto) { 13 plan skip_all => "Can't create temporary test database: $@"; 14 exit 0; 15} 16 17use File::Temp; 18my %mogroot; 19$mogroot{1} = File::Temp::tempdir( CLEANUP => 1 ); 20$mogroot{2} = File::Temp::tempdir( CLEANUP => 1 ); 21my $dev2host = { 1 => 1, 2 => 2, }; 22foreach (sort { $a <=> $b } keys %$dev2host) { 23 my $root = $mogroot{$dev2host->{$_}}; 24 mkdir("$root/dev$_") or die "Failed to create dev$_ dir: $!"; 25} 26 27my $ms1 = create_mogstored("127.0.1.1", $mogroot{1}); 28ok($ms1, "got mogstored1"); 29my $ms2 = create_mogstored("127.0.1.2", $mogroot{2}); 30ok($ms1, "got mogstored2"); 31 32try_for(30, sub { 33 print "Waiting on usage...\n"; 34 -e "$mogroot{1}/dev1/usage" && -e "$mogroot{2}/dev2/usage"; 35}); 36 37my $tmptrack = create_temp_tracker($sto); 38ok($tmptrack); 39 40my $admin = IO::Socket::INET->new(PeerAddr => '127.0.0.1:7001'); 41$admin or die "failed to create admin socket: $!"; 42my $moga = MogileFS::Admin->new(hosts => [ "127.0.0.1:7001" ]); 43my $mogc = MogileFS::Client->new( 44 domain => "testdom", 45 hosts => [ "127.0.0.1:7001" ], 46 ); 47my $be = $mogc->{backend}; # gross, reaching inside of MogileFS::Client 48 49# test some basic commands to backend 50ok($tmptrack->mogadm("domain", "add", "testdom"), "created test domain"); 51ok($tmptrack->mogadm("class", "add", "testdom", "2copies", "--mindevcount=2"), "created 2copies class in testdom"); 52ok($tmptrack->mogadm("class", "add", "testdom", "1copy", "--mindevcount=1"), "created 1copy class in testdom"); 53 54ok($tmptrack->mogadm("host", "add", "hostA", "--ip=127.0.1.1", "--status=alive"), "created hostA"); 55ok($tmptrack->mogadm("host", "add", "hostB", "--ip=127.0.1.2", "--status=alive"), "created hostB"); 56 57ok($tmptrack->mogadm("device", "add", "hostA", 1), "created dev1 on hostA"); 58ok($tmptrack->mogadm("device", "add", "hostB", 2), "created dev2 on hostB"); 59 60sub wait_for_monitor { 61 my $be = shift; 62 my $was = $be->{timeout}; # can't use local on phash :( 63 $be->{timeout} = 10; 64 ok($be->do_request("clear_cache", {}), "waited for monitor") 65 or die "Failed to wait for monitor"; 66 ok($be->do_request("clear_cache", {}), "waited for monitor") 67 or die "Failed to wait for monitor"; 68 $be->{timeout} = $was; 69} 70 71sub full_fsck { 72 my $tmptrack = shift; 73 74 ok($tmptrack->mogadm("fsck", "stop"), "stop fsck"); 75 ok($tmptrack->mogadm("fsck", "clearlog"), "clear fsck log"); 76 ok($tmptrack->mogadm("fsck", "reset"), "reset fsck"); 77 ok($tmptrack->mogadm("fsck", "start"), "started fsck"); 78} 79 80wait_for_monitor($be); 81 82my ($req, $rv, %opts, @paths, @fsck_log); 83my $ua = LWP::UserAgent->new; 84 85use Data::Dumper; 86use Digest::MD5 qw/md5_hex/; 87 88# verify upload checksum 89{ 90 my $key = "ok"; 91 %opts = ( domain => "testdom", class => "1copy", key => $key ); 92 $rv = $be->do_request("create_open", \%opts); 93 %opts = %$rv; 94 ok($rv && $rv->{path}, "create_open succeeded"); 95 $req = HTTP::Request->new(PUT => $rv->{path}); 96 $req->content("blah"); 97 $rv = $ua->request($req); 98 ok($rv->is_success, "PUT successful"); 99 $opts{key} = $key; 100 $opts{domain} = "testdom"; 101 $opts{checksum} = "MD5:".md5_hex('blah'); 102 $opts{checksumverify} = 1; 103 $rv = $be->do_request("create_close", \%opts); 104 ok($rv, "checksum verified successfully"); 105 is($sto->get_checksum($opts{fid}), undef, "checksum not saved"); 106 ok($mogc->file_info($key), "file_info($key) is sane"); 107} 108 109# corrupted upload checksum fails 110{ 111 my $key = 'corrupt'; 112 %opts = ( domain => "testdom", class => "1copy", key => $key ); 113 $rv = $be->do_request("create_open", \%opts); 114 %opts = %$rv; 115 ok($rv && $rv->{path}, "create_open succeeded"); 116 $req = HTTP::Request->new(PUT => $rv->{path}); 117 $req->content("blah"); 118 $rv = $ua->request($req); 119 ok($rv->is_success, "PUT successful"); 120 $opts{key} = $key; 121 $opts{domain} = "testdom"; 122 123 $opts{checksumverify} = 1; 124 $opts{checksum} = "MD5:".md5_hex('fail'); 125 $rv = $be->do_request("create_close", \%opts); 126 ok(!defined($rv), "checksum verify noticed mismatch"); 127 my $hex = md5_hex('blah'); 128 is('checksum_mismatch', $be->{lasterr}, "error code is correct"); 129 ok($be->{lasterrstr} =~ /actual: MD5:$hex;/, "error message shows actual:"); 130 is($sto->get_checksum($opts{fid}), undef, "checksum not saved"); 131 is($mogc->file_info($key), undef, "$key not uploaded"); 132} 133 134# enable saving MD5 checksums in "2copies" class 135{ 136 %opts = ( domain => "testdom", class => "2copies", 137 hashtype => "MD5", mindevcount => 2 ); 138 ok($be->do_request("update_class", \%opts), "update class"); 139 wait_for_monitor($be); 140} 141 142# save new row to checksum table 143{ 144 my $key = 'savecksum'; 145 146 want($admin, 0, "replicate"); 147 148 %opts = ( domain => "testdom", class => "2copies", key => $key ); 149 $rv = $be->do_request("create_open", \%opts); 150 %opts = %$rv; 151 ok($rv && $rv->{path}, "create_open succeeded"); 152 $req = HTTP::Request->new(PUT => $rv->{path}); 153 $req->content("blah"); 154 $rv = $ua->request($req); 155 ok($rv->is_success, "PUT successful"); 156 $opts{key} = $key; 157 $opts{domain} = "testdom"; 158 $opts{checksum} = "MD5:".md5_hex('blah'); 159 $opts{checksumverify} = 1; 160 $rv = $be->do_request("create_close", \%opts); 161 ok($rv, "checksum verified successfully"); 162 my $row = $sto->get_checksum($opts{fid}); 163 ok($row, "checksum saved"); 164 my $info = $mogc->file_info($key); 165 ok($info, "file_info($key) is sane"); 166 is($info->{checksum}, "MD5:".md5_hex('blah'), 'checksum shows up'); 167 is($sto->delete_checksum($info->{fid}), 1, "$key checksum row deleted"); 168 $info = $mogc->file_info($key); 169 is($info->{checksum}, "MISSING", 'checksum is MISSING after delete'); 170 171 want($admin, 1, "replicate"); 172 173 # wait for replicate to recreate checksum 174 try_for(30, sub { 175 @paths = $mogc->get_paths($key); 176 scalar(@paths) != 1; 177 }); 178 is(scalar(@paths), 2, "replicate successfully with good checksum"); 179 180 $info = $mogc->file_info($key); 181 is($info->{checksum}, "MD5:".md5_hex('blah'), 'checksum recreated on repl'); 182} 183 184# flip checksum classes around 185{ 186 my @classes; 187 %opts = ( domain => "testdom", class => "1copy", mindevcount => 1 ); 188 189 $opts{hashtype} = "NONE"; 190 ok($be->do_request("update_class", \%opts), "update class"); 191 @classes = grep { $_->{classname} eq '1copy' } $sto->get_all_classes; 192 is($classes[0]->{hashtype}, undef, "hashtype unset"); 193 194 $opts{hashtype} = "MD5"; 195 ok($be->do_request("update_class", \%opts), "update class"); 196 @classes = grep { $_->{classname} eq '1copy' } $sto->get_all_classes; 197 is($classes[0]->{hashtype}, 1, "hashtype is 1 (MD5)"); 198 199 $opts{hashtype} = "NONE"; 200 ok($be->do_request("update_class", \%opts), "update class"); 201 @classes = grep { $_->{classname} eq '1copy' } $sto->get_all_classes; 202 is($classes[0]->{hashtype}, undef, "hashtype unset"); 203} 204 205# save checksum on replicate, client didn't care to provide one 206{ 207 my $key = 'lazycksum'; 208 209 want($admin, 0, "replicate"); 210 211 my $fh = $mogc->new_file($key, "2copies"); 212 print $fh "lazy"; 213 ok(close($fh), "closed file"); 214 my $info = $mogc->file_info($key); 215 is($info->{checksum}, 'MISSING', 'checksum is MISSING'); 216 217 want($admin, 1, "replicate"); 218 219 try_for(30, sub { 220 @paths = $mogc->get_paths($key); 221 scalar(@paths) != 1; 222 }); 223 is(scalar(@paths), 2, "replicate successfully with good checksum"); 224 225 $info = $mogc->file_info($key); 226 is($info->{checksum}, "MD5:".md5_hex("lazy"), 'checksum is set after repl'); 227} 228 229# fsck recreates checksum when missing 230{ 231 my $key = 'lazycksum'; 232 my $info = $mogc->file_info($key); 233 $sto->delete_checksum($info->{fid}); 234 $info = $mogc->file_info($key); 235 is($info->{checksum}, "MISSING", "checksum is missing"); 236 full_fsck($tmptrack); 237 238 try_for(30, sub { 239 $info = $mogc->file_info($key); 240 $info->{checksum} ne "MISSING"; 241 }); 242 is($info->{checksum}, "MD5:".md5_hex("lazy"), 'checksum is set after fsck'); 243 244 @fsck_log = $sto->fsck_log_rows; 245 is(scalar(@fsck_log), 1, "fsck log has one row"); 246 is($fsck_log[0]->{fid}, $info->{fid}, "fid matches in fsck log"); 247 is($fsck_log[0]->{evcode}, "NSUM", "missing checksum logged"); 248} 249 250# fsck fixes a file corrupt file 251{ 252 my $key = 'lazycksum'; 253 my $info = $mogc->file_info($key); 254 @paths = $mogc->get_paths($key); 255 is(scalar(@paths), 2, "2 paths for lazycksum"); 256 $req = HTTP::Request->new(PUT => $paths[0]); 257 $req->content("LAZY"); 258 $rv = $ua->request($req); 259 ok($rv->is_success, "upload to corrupt a file successful"); 260 is($ua->get($paths[0])->content, "LAZY", "file successfully corrupted"); 261 is($ua->get($paths[1])->content, "lazy", "paths[1] not corrupted"); 262 263 full_fsck($tmptrack); 264 265 try_for(30, sub { 266 @fsck_log = $sto->fsck_log_rows; 267 scalar(@fsck_log) != 0; 268 }); 269 270 is(scalar(@fsck_log), 1, "fsck log has one row"); 271 is($fsck_log[0]->{fid}, $info->{fid}, "fid matches in fsck log"); 272 is($fsck_log[0]->{evcode}, "REPL", "repl for mismatched checksum logged"); 273 274 try_for(30, sub { 275 @paths = $mogc->get_paths($key); 276 scalar(@paths) >= 2; 277 }); 278 279 is(scalar(@paths), 2, "2 paths for key after replication"); 280 is($ua->get($paths[0])->content, "lazy", "paths[0] is correct"); 281 is($ua->get($paths[1])->content, "lazy", "paths[1] is correct"); 282 $info = $mogc->file_info($key); 283 is($info->{checksum}, "MD5:".md5_hex("lazy"), 'checksum unchanged after fsck'); 284} 285 286# fsck notices when all files are corrupt 287{ 288 my $key = 'lazycksum'; 289 my $info = $mogc->file_info($key); 290 @paths = $mogc->get_paths($key); 291 is(scalar(@paths), 2, "2 paths for lazycksum"); 292 293 $req = HTTP::Request->new(PUT => $paths[0]); 294 $req->content("0000"); 295 $rv = $ua->request($req); 296 ok($rv->is_success, "upload to corrupt a file successful"); 297 is($ua->get($paths[0])->content, "0000", "successfully corrupted"); 298 299 $req = HTTP::Request->new(PUT => $paths[1]); 300 $req->content("1111"); 301 $rv = $ua->request($req); 302 ok($rv->is_success, "upload to corrupt a file successful"); 303 is($ua->get($paths[1])->content, "1111", "successfully corrupted"); 304 305 full_fsck($tmptrack); 306 307 try_for(30, sub { 308 @fsck_log = $sto->fsck_log_rows; 309 scalar(@fsck_log) != 0; 310 }); 311 312 is(scalar(@fsck_log), 1, "fsck log has one row"); 313 is($fsck_log[0]->{fid}, $info->{fid}, "fid matches in fsck log"); 314 is($fsck_log[0]->{evcode}, "BSUM", "BSUM logged"); 315 316 @paths = $mogc->get_paths($key); 317 is(scalar(@paths), 2, "2 paths for checksum"); 318 @paths = sort( map { $ua->get($_)->content } @paths); 319 is(join(', ', @paths), "0000, 1111", "corrupted content preserved"); 320} 321 322# reuploaded checksum row clobbers old checksum 323{ 324 my $key = 'lazycksum'; 325 my $info = $mogc->file_info($key); 326 327 ok($sto->get_checksum($info->{fid}), "old checksum row exists"); 328 329 my $fh = $mogc->new_file($key, "2copies"); 330 print $fh "HAZY"; 331 ok(close($fh), "closed replacement file (lazycksum => HAZY)"); 332 333 try_for(30, sub { ! $sto->get_checksum($info->{fid}); }); 334 is($sto->get_checksum($info->{fid}), undef, "old checksum is gone"); 335} 336 337# completely corrupted files with no checksum row 338{ 339 my $key = 'lazycksum'; 340 try_for(30, sub { 341 @paths = $mogc->get_paths($key); 342 scalar(@paths) >= 2; 343 }); 344 is(scalar(@paths), 2, "replicated succesfully"); 345 346 my $info = $mogc->file_info($key); 347 is($info->{checksum}, "MD5:".md5_hex("HAZY"), "checksum created on repl"); 348 349 $req = HTTP::Request->new(PUT => $paths[0]); 350 $req->content("MAYB"); 351 $rv = $ua->request($req); 352 ok($rv->is_success, "upload to corrupt a file successful"); 353 is($ua->get($paths[0])->content, "MAYB", "successfully corrupted (MAYB)"); 354 355 $req = HTTP::Request->new(PUT => $paths[1]); 356 $req->content("CRZY"); 357 $rv = $ua->request($req); 358 ok($rv->is_success, "upload to corrupt a file successful"); 359 is($ua->get($paths[1])->content, "CRZY", "successfully corrupted (CRZY)"); 360 361 is($sto->delete_checksum($info->{fid}), 1, "nuke new checksum"); 362 $info = $mogc->file_info($key); 363 is($info->{checksum}, "MISSING", "checksum is MISSING"); 364 365 full_fsck($tmptrack); 366 367 try_for(30, sub { 368 @fsck_log = $sto->fsck_log_rows; 369 scalar(@fsck_log) != 0; 370 }); 371 372 is(scalar(@fsck_log), 1, "fsck log has one row"); 373 is($fsck_log[0]->{fid}, $info->{fid}, "fid matches in fsck log"); 374 is($fsck_log[0]->{evcode}, "BSUM", "BSUM logged"); 375} 376 377# disable MD5 checksums in "2copies" class 378{ 379 %opts = ( domain => "testdom", class => "2copies", 380 hashtype => "NONE", mindevcount => 2 ); 381 ok($be->do_request("update_class", \%opts), "update class"); 382 wait_for_monitor($be); 383} 384 385# use fsck_checksum=MD5 instead of per-class checksums 386{ 387 my $key = 'lazycksum'; 388 my $info = $mogc->file_info($key); 389 $sto->delete_checksum($info->{fid}); 390 391 ok($tmptrack->mogadm("settings", "set", "fsck_checksum", "MD5"), "enable fsck_checksum=MD5"); 392 wait_for_monitor($be); 393 full_fsck($tmptrack); 394 395 try_for(30, sub { 396 @fsck_log = $sto->fsck_log_rows; 397 scalar(@fsck_log) != 0; 398 }); 399 is(scalar(@fsck_log), 1, "fsck log has one row"); 400 is($fsck_log[0]->{fid}, $info->{fid}, "fid matches in fsck log"); 401 is($fsck_log[0]->{evcode}, "MSUM", "MSUM logged"); 402} 403 404# ensure server setting is visible 405use MogileFS::Admin; 406{ 407 my $settings = $moga->server_settings; 408 is($settings->{fsck_checksum}, 'MD5', "fsck_checksum server setting visible"); 409} 410 411use MogileFS::Config; 412 413# disable checksumming entirely, regardless of class setting 414{ 415 %opts = ( domain => "testdom", class => "2copies", 416 hashtype => "MD5", mindevcount => 2 ); 417 ok($be->do_request("update_class", \%opts), "update class"); 418 wait_for_monitor($be); 419 420 ok($tmptrack->mogadm("settings", "set", "fsck_checksum", "off"), "set fsck_checksum=off"); 421 wait_for_monitor($be); 422 my $settings = $moga->server_settings; 423 is($settings->{fsck_checksum}, 'off', "fsck_checksum server setting visible"); 424 full_fsck($tmptrack); 425 my $nr; 426 try_for(1000, sub { 427 $nr = $sto->file_queue_length(FSCK_QUEUE); 428 $nr eq '0'; 429 }); 430 is($nr, '0', "fsck finished"); 431 @fsck_log = $sto->fsck_log_rows; 432 is(scalar(@fsck_log), 0, "fsck log is empty with fsck_checksum=off"); 433} 434 435# set fsck_checksum=class and ensure that works again 436{ 437 my $info = $mogc->file_info('lazycksum'); 438 ok($tmptrack->mogadm("settings", "set", "fsck_checksum", "class"), "set fsck_checksum=class"); 439 wait_for_monitor($be); 440 my $settings = $moga->server_settings; 441 ok(! defined($settings->{fsck_checksum}), "fsck_checksum=class server setting hidden (default)"); 442 full_fsck($tmptrack); 443 444 try_for(30, sub { 445 @fsck_log = $sto->fsck_log_rows; 446 scalar(@fsck_log) != 0; 447 }); 448 449 is(scalar(@fsck_log), 1, "fsck log has one row"); 450 is($fsck_log[0]->{fid}, $info->{fid}, "fid matches in fsck log"); 451 is($fsck_log[0]->{evcode}, "BSUM", "BSUM logged"); 452} 453 454done_testing(); 455