1#!/usr/bin/perl 2 3=head1 NAME 4 5moglistfids -- Iterate fid/key data from a MogileFS installation 6 7=head1 DESCRIPTION 8 9Example utility for pulling all file data out of a MogileFS installation. 10Utilities like this can be built on for creating backup systems, 11Mogile<->Mogile syncronization systems, or Mogile->S3 syncronization. 12 13This method is only a way of pulling new files which have existed since the 14last time it was checked, as there's no logging of deleted files. 15 16=head1 OPTIONS 17 18=over 19 20=item --trackers=host1:7001,host2:7001 21 22Use these MogileFS trackers to negotiate with. 23 24=item --fromfid=<fid> 25 26The highest numbered fid fetched the last time this utility was run. 27 28=item --count=<count> 29 30Numer of fids to inspect and return. 31 32=back 33 34=head1 AUTHOR 35 36Dormando E<lt>L<dormando@rydia.net>E<gt> 37 38=head1 BUGS 39 40None known. 41 42=head1 LICENSE 43 44Licensed for use and redistribution under the same terms as Perl itself. 45 46=cut 47 48use strict; 49use warnings; 50 51use MogileFS::Admin; 52use lib './lib'; 53use MogileFS::Utils; 54 55my $util = MogileFS::Utils->new; 56my $usage = "--trackers=host --fromfid=123 --count=5000"; 57my $c = $util->getopts($usage, qw/fromfid=i count=i/); 58 59my $moga = MogileFS::Admin->new(hosts => $c->{trackers}); 60 61my $fromfid = $c->{fromfid} || 0; 62my $count = $c->{count} || 100; 63 64while ($count) { 65 # Try to fetch the max, but we will likely get less. 66 my $fids_chunk = $moga->list_fids($fromfid, $count); 67 if ($moga->errcode) { 68 die "Error listing fids: ", $moga->errstr, "\n"; 69 } 70 my @fids = sort { $a <=> $b } keys %$fids_chunk; 71 last unless @fids; 72 $fromfid = $fids[-1]; 73 $count -= @fids; 74 for my $fid (@fids) { 75 my $file = $fids_chunk->{$fid}; 76 print "fid ", $fid, "\n"; 77 for my $key (sort keys %$file) { 78 my $val = $file->{$key}; 79 $val = _escape_url_string($val) if $key eq 'dkey'; 80 print $key, " ", $val, "\n"; 81 } 82 print "\n"; 83 } 84} 85 86sub _escape_url_string { 87 my $str = shift; 88 $str =~ s/([^a-zA-Z0-9_\,\-.\/\\\: ])/uc sprintf("%%%02x",ord($1))/eg; 89 $str =~ tr/ /+/; 90 return $str; 91} 92