1# BEGIN BPS TAGGED BLOCK {{{
2#
3# COPYRIGHT:
4#
5# This software is Copyright (c) 1996-2021 Best Practical Solutions, LLC
6#                                          <sales@bestpractical.com>
7#
8# (Except where explicitly superseded by other copyright notices)
9#
10#
11# LICENSE:
12#
13# This work is made available to you under the terms of Version 2 of
14# the GNU General Public License. A copy of that license should have
15# been provided with this software, but in any event can be snarfed
16# from www.gnu.org.
17#
18# This work is distributed in the hope that it will be useful, but
19# WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
21# General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
26# 02110-1301 or visit their web page on the internet at
27# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
28#
29#
30# CONTRIBUTION SUBMISSION POLICY:
31#
32# (The following paragraph is not intended to limit the rights granted
33# to you to modify and distribute this software under the terms of
34# the GNU General Public License and is only of importance to you if
35# you choose to contribute your changes and enhancements to the
36# community by submitting them to Best Practical Solutions, LLC.)
37#
38# By intentionally submitting any modifications, corrections or
39# derivatives to this work, or any other work intended for use with
40# Request Tracker, to Best Practical Solutions, LLC, you confirm that
41# you are the copyright holder for those contributions and you grant
42# Best Practical Solutions,  LLC a nonexclusive, worldwide, irrevocable,
43# royalty-free, perpetual, license to use, copy, create derivative
44# works based on those contributions, and sublicense and distribute
45# those contributions and any derivatives thereof.
46#
47# END BPS TAGGED BLOCK }}}
48
49package RT::Migrate::Importer::File;
50
51use strict;
52use warnings;
53use base qw(RT::Migrate::Importer);
54
55sub Init {
56    my $self = shift;
57    my %args = (
58        Directory => undef,
59        Resume    => undef,
60        @_
61    );
62
63    # Directory is required
64    die "Directory is required" unless $args{Directory};
65    die "Invalid path $args{Directory}" unless -d $args{Directory};
66    $self->{Directory} = $args{Directory};
67
68    # Load metadata, if present
69    if (-e "$args{Directory}/rt-serialized") {
70        my $dat = eval { Storable::retrieve("$args{Directory}/rt-serialized"); }
71            or die "Failed to load metadata" . ($@ ? ": $@" : "");
72        $self->LoadMetadata($dat);
73    }
74
75    # Support resuming
76    $self->{Statefile}  = $args{Statefile} || "$args{Directory}/partial-import";
77    unlink $self->{Statefile}
78        if -f $self->{Statefile} and not $args{Resume};
79
80    return $self->SUPER::Init(@_);
81}
82
83sub Import {
84    my $self = shift;
85    my $dir = $self->{Directory};
86
87    if ($self->{Metadata} and $self->{Metadata}{Files}) {
88        $self->{Files} = [ map {s|^.*/|$dir/|;$_} @{$self->{Metadata}{Files}} ];
89    } else {
90        $self->{Files} = [ <$dir/*.dat> ];
91    }
92    $self->{Files} = [ map {File::Spec->rel2abs($_)} @{ $self->{Files} } ];
93
94    $self->RestoreState( $self->{Statefile} );
95
96    local $SIG{  INT  } = sub { $self->{INT} = 1 };
97    local $SIG{__DIE__} = sub { warn "\n", @_; $self->SaveState; exit 1 };
98
99    $self->{Progress}->(undef) if $self->{Progress};
100    while (@{$self->{Files}}) {
101        $self->{Filename} = shift @{$self->{Files}};
102        open(my $fh, "<", $self->{Filename})
103            or die "Can't read $self->{Filename}: $!";
104        if ($self->{Seek}) {
105            seek($fh, $self->{Seek}, 0)
106                or die "Can't seek to $self->{Seek} in $self->{Filename}";
107            $self->{Seek} = undef;
108        }
109        while (not eof($fh)) {
110            $self->{Position} = tell($fh);
111
112            # Stop when we're at a good stopping point
113            die "Caught interrupt, quitting.\n" if $self->{INT};
114
115            $self->ReadStream( $fh );
116        }
117    }
118
119    $self->CloseStream;
120
121    # Return creation counts
122    return $self->ObjectCount;
123}
124
125sub List {
126    my $self = shift;
127    my $dir = $self->{Directory};
128
129    my %found = ( "RT::System" => 1 );
130    my @files = ($self->{Metadata} and $self->{Metadata}{Files}) ?
131        @{ $self->{Metadata}{Files} } : <$dir/*.dat>;
132    @files = map {File::Spec->rel2abs($_)} @files;
133
134    for my $filename (@files) {
135        open(my $fh, "<", $filename)
136            or die "Can't read $filename: $!";
137        while (not eof($fh)) {
138            my $loaded = Storable::fd_retrieve($fh);
139            if (ref $loaded eq "HASH") {
140                $self->LoadMetadata( $loaded );
141                next;
142            }
143
144            if ($self->{DumpObjects}) {
145                print STDERR Data::Dumper::Dumper($loaded), "\n"
146                    if $self->{DumpObjects}{ $loaded->[0] };
147            }
148
149            my ($class, $uid, $data) = @{$loaded};
150            $self->{ObjectCount}{$class}++;
151            $found{$uid} = 1;
152            delete $self->{Pending}{$uid};
153            for (grep {ref $data->{$_}} keys %{$data}) {
154                my $uid_ref = ${ $data->{$_} };
155                unless (defined $uid_ref) {
156                    push @{ $self->{Invalid} }, { uid => $uid, column => $_ };
157                    next;
158                }
159                next if $found{$uid_ref};
160                next if $uid_ref =~ /^RT::Principal-/;
161                push @{$self->{Pending}{$uid_ref} ||= []}, {uid => $uid};
162            }
163        }
164    }
165
166    return $self->ObjectCount;
167}
168
169sub RestoreState {
170    my $self = shift;
171    my ($statefile) = @_;
172    return unless $statefile && -f $statefile;
173
174    my $state = Storable::retrieve( $self->{Statefile} );
175    $self->{$_} = $state->{$_} for keys %{$state};
176    unlink $self->{Statefile};
177
178    print STDERR "Resuming partial import...\n";
179    sleep 2;
180    return 1;
181}
182
183sub SaveState {
184    my $self = shift;
185
186    my %data;
187    unshift @{$self->{Files}}, $self->{Filename};
188    $self->{Seek} = $self->{Position};
189    $data{$_} = $self->{$_} for
190        qw/Filename Seek Position Files
191           Organization ObjectCount
192           NewQueues NewCFs
193           SkipTransactions Pending Invalid
194           UIDs
195           OriginalId ExcludeOrganization Clone
196          /;
197    Storable::nstore(\%data, $self->{Statefile});
198
199    print STDERR <<EOT;
200
201Importer state has been written to the file:
202    $self->{Statefile}
203
204It may be possible to resume the import by re-running rt-importer.
205EOT
206}
207
2081;
209