1#!/usr/bin/perl
2
3=head1 NAME
4
5moglistfids -- Iterate fid/key data from a MogileFS installation
6
7=head1 DESCRIPTION
8
9Example utility for pulling all file data out of a MogileFS installation.
10Utilities like this can be built on for creating backup systems,
11Mogile<->Mogile syncronization systems, or Mogile->S3 syncronization.
12
13This method is only a way of pulling new files which have existed since the
14last time it was checked, as there's no logging of deleted files.
15
16=head1 OPTIONS
17
18=over
19
20=item --trackers=host1:7001,host2:7001
21
22Use these MogileFS trackers to negotiate with.
23
24=item --fromfid=<fid>
25
26The highest numbered fid fetched the last time this utility was run.
27
28=item --count=<count>
29
30Numer of fids to inspect and return.
31
32=back
33
34=head1 AUTHOR
35
36Dormando E<lt>L<dormando@rydia.net>E<gt>
37
38=head1 BUGS
39
40None known.
41
42=head1 LICENSE
43
44Licensed for use and redistribution under the same terms as Perl itself.
45
46=cut
47
48use strict;
49use warnings;
50
51use MogileFS::Admin;
52use lib './lib';
53use MogileFS::Utils;
54
55my $util = MogileFS::Utils->new;
56my $usage = "--trackers=host --fromfid=123 --count=5000";
57my $c = $util->getopts($usage, qw/fromfid=i count=i/);
58
59my $moga = MogileFS::Admin->new(hosts => $c->{trackers});
60
61my $fromfid = $c->{fromfid} || 0;
62my $count   = $c->{count} || 100;
63
64while ($count) {
65    # Try to fetch the max, but we will likely get less.
66    my $fids_chunk = $moga->list_fids($fromfid, $count);
67    if ($moga->errcode) {
68        die "Error listing fids: ", $moga->errstr, "\n";
69    }
70    my @fids = sort { $a <=> $b } keys %$fids_chunk;
71    last unless @fids;
72    $fromfid = $fids[-1];
73    $count  -= @fids;
74    for my $fid (@fids) {
75        my $file = $fids_chunk->{$fid};
76        print "fid ", $fid, "\n";
77        for my $key (sort keys %$file) {
78            my $val = $file->{$key};
79            $val = _escape_url_string($val) if $key eq 'dkey';
80            print $key, " ", $val, "\n";
81        }
82        print "\n";
83    }
84}
85
86sub _escape_url_string {
87    my $str = shift;
88    $str =~ s/([^a-zA-Z0-9_\,\-.\/\\\: ])/uc sprintf("%%%02x",ord($1))/eg;
89    $str =~ tr/ /+/;
90    return $str;
91}
92