1# BEGIN BPS TAGGED BLOCK {{{ 2# 3# COPYRIGHT: 4# 5# This software is Copyright (c) 1996-2021 Best Practical Solutions, LLC 6# <sales@bestpractical.com> 7# 8# (Except where explicitly superseded by other copyright notices) 9# 10# 11# LICENSE: 12# 13# This work is made available to you under the terms of Version 2 of 14# the GNU General Public License. A copy of that license should have 15# been provided with this software, but in any event can be snarfed 16# from www.gnu.org. 17# 18# This work is distributed in the hope that it will be useful, but 19# WITHOUT ANY WARRANTY; without even the implied warranty of 20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21# General Public License for more details. 22# 23# You should have received a copy of the GNU General Public License 24# along with this program; if not, write to the Free Software 25# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 26# 02110-1301 or visit their web page on the internet at 27# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html. 28# 29# 30# CONTRIBUTION SUBMISSION POLICY: 31# 32# (The following paragraph is not intended to limit the rights granted 33# to you to modify and distribute this software under the terms of 34# the GNU General Public License and is only of importance to you if 35# you choose to contribute your changes and enhancements to the 36# community by submitting them to Best Practical Solutions, LLC.) 37# 38# By intentionally submitting any modifications, corrections or 39# derivatives to this work, or any other work intended for use with 40# Request Tracker, to Best Practical Solutions, LLC, you confirm that 41# you are the copyright holder for those contributions and you grant 42# Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable, 43# royalty-free, perpetual, license to use, copy, create derivative 44# works based on those contributions, and sublicense and distribute 45# those contributions and any derivatives thereof. 46# 47# END BPS TAGGED BLOCK }}} 48 49package RT::Migrate::Importer::File; 50 51use strict; 52use warnings; 53use base qw(RT::Migrate::Importer); 54 55sub Init { 56 my $self = shift; 57 my %args = ( 58 Directory => undef, 59 Resume => undef, 60 @_ 61 ); 62 63 # Directory is required 64 die "Directory is required" unless $args{Directory}; 65 die "Invalid path $args{Directory}" unless -d $args{Directory}; 66 $self->{Directory} = $args{Directory}; 67 68 # Load metadata, if present 69 if (-e "$args{Directory}/rt-serialized") { 70 my $dat = eval { Storable::retrieve("$args{Directory}/rt-serialized"); } 71 or die "Failed to load metadata" . ($@ ? ": $@" : ""); 72 $self->LoadMetadata($dat); 73 } 74 75 # Support resuming 76 $self->{Statefile} = $args{Statefile} || "$args{Directory}/partial-import"; 77 unlink $self->{Statefile} 78 if -f $self->{Statefile} and not $args{Resume}; 79 80 return $self->SUPER::Init(@_); 81} 82 83sub Import { 84 my $self = shift; 85 my $dir = $self->{Directory}; 86 87 if ($self->{Metadata} and $self->{Metadata}{Files}) { 88 $self->{Files} = [ map {s|^.*/|$dir/|;$_} @{$self->{Metadata}{Files}} ]; 89 } else { 90 $self->{Files} = [ <$dir/*.dat> ]; 91 } 92 $self->{Files} = [ map {File::Spec->rel2abs($_)} @{ $self->{Files} } ]; 93 94 $self->RestoreState( $self->{Statefile} ); 95 96 local $SIG{ INT } = sub { $self->{INT} = 1 }; 97 local $SIG{__DIE__} = sub { warn "\n", @_; $self->SaveState; exit 1 }; 98 99 $self->{Progress}->(undef) if $self->{Progress}; 100 while (@{$self->{Files}}) { 101 $self->{Filename} = shift @{$self->{Files}}; 102 open(my $fh, "<", $self->{Filename}) 103 or die "Can't read $self->{Filename}: $!"; 104 if ($self->{Seek}) { 105 seek($fh, $self->{Seek}, 0) 106 or die "Can't seek to $self->{Seek} in $self->{Filename}"; 107 $self->{Seek} = undef; 108 } 109 while (not eof($fh)) { 110 $self->{Position} = tell($fh); 111 112 # Stop when we're at a good stopping point 113 die "Caught interrupt, quitting.\n" if $self->{INT}; 114 115 $self->ReadStream( $fh ); 116 } 117 } 118 119 $self->CloseStream; 120 121 # Return creation counts 122 return $self->ObjectCount; 123} 124 125sub List { 126 my $self = shift; 127 my $dir = $self->{Directory}; 128 129 my %found = ( "RT::System" => 1 ); 130 my @files = ($self->{Metadata} and $self->{Metadata}{Files}) ? 131 @{ $self->{Metadata}{Files} } : <$dir/*.dat>; 132 @files = map {File::Spec->rel2abs($_)} @files; 133 134 for my $filename (@files) { 135 open(my $fh, "<", $filename) 136 or die "Can't read $filename: $!"; 137 while (not eof($fh)) { 138 my $loaded = Storable::fd_retrieve($fh); 139 if (ref $loaded eq "HASH") { 140 $self->LoadMetadata( $loaded ); 141 next; 142 } 143 144 if ($self->{DumpObjects}) { 145 print STDERR Data::Dumper::Dumper($loaded), "\n" 146 if $self->{DumpObjects}{ $loaded->[0] }; 147 } 148 149 my ($class, $uid, $data) = @{$loaded}; 150 $self->{ObjectCount}{$class}++; 151 $found{$uid} = 1; 152 delete $self->{Pending}{$uid}; 153 for (grep {ref $data->{$_}} keys %{$data}) { 154 my $uid_ref = ${ $data->{$_} }; 155 unless (defined $uid_ref) { 156 push @{ $self->{Invalid} }, { uid => $uid, column => $_ }; 157 next; 158 } 159 next if $found{$uid_ref}; 160 next if $uid_ref =~ /^RT::Principal-/; 161 push @{$self->{Pending}{$uid_ref} ||= []}, {uid => $uid}; 162 } 163 } 164 } 165 166 return $self->ObjectCount; 167} 168 169sub RestoreState { 170 my $self = shift; 171 my ($statefile) = @_; 172 return unless $statefile && -f $statefile; 173 174 my $state = Storable::retrieve( $self->{Statefile} ); 175 $self->{$_} = $state->{$_} for keys %{$state}; 176 unlink $self->{Statefile}; 177 178 print STDERR "Resuming partial import...\n"; 179 sleep 2; 180 return 1; 181} 182 183sub SaveState { 184 my $self = shift; 185 186 my %data; 187 unshift @{$self->{Files}}, $self->{Filename}; 188 $self->{Seek} = $self->{Position}; 189 $data{$_} = $self->{$_} for 190 qw/Filename Seek Position Files 191 Organization ObjectCount 192 NewQueues NewCFs 193 SkipTransactions Pending Invalid 194 UIDs 195 OriginalId ExcludeOrganization Clone 196 /; 197 Storable::nstore(\%data, $self->{Statefile}); 198 199 print STDERR <<EOT; 200 201Importer state has been written to the file: 202 $self->{Statefile} 203 204It may be possible to resume the import by re-running rt-importer. 205EOT 206} 207 2081; 209