1package File::Map; 2$File::Map::VERSION = '0.67'; 3# This software is copyright (c) 2008, 2009, 2010, 2011, 2012 by Leon Timmermans <leont@cpan.org>. 4# 5# This is free software; you can redistribute it and/or modify it under 6# the same terms as perl itself. 7 8use 5.008; 9use strict; 10use warnings FATAL => 'all'; 11use subs qw{PROT_READ PROT_WRITE MAP_PRIVATE MAP_SHARED MAP_FILE MAP_ANONYMOUS}; 12 13use Sub::Exporter::Progressive 0.001005 (); 14use XSLoader (); 15use Carp qw/croak carp/; 16use PerlIO::Layers qw/query_handle/; 17 18XSLoader::load('File::Map', File::Map->VERSION); 19 20my %export_data = ( 21 'map' => [qw/map_handle map_file map_anonymous unmap sys_map/], 22 extra => [qw/remap sync pin unpin advise protect/], 23 'lock' => [qw/wait_until notify broadcast lock_map/], 24 constants => [qw/PROT_NONE PROT_READ PROT_WRITE PROT_EXEC MAP_ANONYMOUS MAP_SHARED MAP_PRIVATE MAP_ANON MAP_FILE/] 25); 26 27{ 28 my (@export_ok, %export_tags); 29 30 while (my ($category, $functions) = each %export_data) { 31 for my $function (grep { defined &{$_} } @{$functions}) { 32 push @export_ok, $function; 33 push @{ $export_tags{$category} }, $function; 34 } 35 } 36 37 Sub::Exporter::Progressive->import(-setup => { exports => \@export_ok, groups => \%export_tags }); 38} 39 40my $anon_fh = -1; 41 42sub _check_layers { 43 my $fh = shift; 44 croak "Can't map fake filehandle" if fileno $fh < 0; 45 if (warnings::enabled('layer')) { 46 carp "Shouldn't map non-binary filehandle" if not query_handle($fh, 'mappable'); 47 } 48 return query_handle($fh, 'utf8'); 49} 50 51sub _get_offset_length { 52 my ($offset, $length, $fh) = @_; 53 54 my $size = -s $fh; 55 $offset ||= 0; 56 $length ||= $size - $offset; 57 my $end = $offset + $length; 58 croak "Window ($offset,$end) is outside the file" if $offset < 0 or $end > $size and not -c _; 59 return ($offset, $length); 60} 61 62## no critic (Subroutines::RequireArgUnpacking) 63 64sub map_handle { 65 my (undef, $fh, $mode, $offset, $length) = @_; 66 my $utf8 = _check_layers($fh); 67 ($offset, $length) = _get_offset_length($offset, $length, $fh); 68 _mmap_impl($_[0], $length, _protection_value($mode || '<'), MAP_SHARED | MAP_FILE, fileno $fh, $offset, $utf8); 69 return; 70} 71 72sub map_file { 73 my (undef, $filename, $mode, $offset, $length) = @_; 74 $mode ||= '<'; 75 my ($minimode, $encoding) = $mode =~ / \A ([^:]+) ([:\w-]+)? \z /xms; 76 $encoding = ':raw' if not defined $encoding; 77 open my $fh, $minimode.$encoding, $filename or croak "Couldn't open file $filename: $!"; 78 my $utf8 = _check_layers($fh); 79 ($offset, $length) = _get_offset_length($offset, $length, $fh); 80 _mmap_impl($_[0], $length, _protection_value($minimode), MAP_SHARED | MAP_FILE, fileno $fh, $offset, $utf8); 81 close $fh or croak "Couldn't close $filename after mapping: $!"; 82 return; 83} 84 85my %flag_for = ( 86 private => MAP_PRIVATE, 87 shared => MAP_SHARED, 88); 89sub map_anonymous { 90 my (undef, $length, $flag_name) = @_; 91 my $flag = $flag_for{ $flag_name || 'shared' }; 92 croak "No such flag '$flag_name'" if not defined $flag; 93 croak 'Zero length specified for anonymous map' if $length == 0; 94 _mmap_impl($_[0], $length, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | $flag, $anon_fh, 0); 95 return; 96} 97 98sub sys_map { ## no critic (ProhibitManyArgs) 99 my (undef, $length, $protection, $flags, $fh, $offset) = @_; 100 my $utf8 = _check_layers($fh); 101 my $fd = ($flags & MAP_ANONYMOUS) ? $anon_fh : fileno $fh; 102 $offset ||= 0; 103 _mmap_impl($_[0], $length, $protection, $flags, $fd, $offset, $utf8); 104 return; 105} 106 1071; 108 109#ABSTRACT: Memory mapping made simple and safe. 110 111__END__ 112 113=pod 114 115=encoding UTF-8 116 117=head1 NAME 118 119File::Map - Memory mapping made simple and safe. 120 121=head1 VERSION 122 123version 0.67 124 125=head1 SYNOPSIS 126 127 use File::Map 'map_file'; 128 129 map_file my $map, $filename, '+<'; 130 $map =~ s/bar/quz/g; 131 substr $map, 1024, 11, "Hello world"; 132 133=head1 DESCRIPTION 134 135File::Map maps files or anonymous memory into perl variables. 136 137=head2 Advantages of memory mapping 138 139=over 4 140 141=item * Unlike normal perl variables, mapped memory is (usually) shared between threads or forked processes. 142 143=item * It is an efficient way to slurp an entire file. Unlike for example L<File::Slurp>, this module returns almost immediately, loading the pages lazily on access. This means you only 'pay' for the parts of the file you actually use. 144 145=item * Perl usually doesn't return memory to the system while running, mapped memory can be returned. 146 147=back 148 149=head2 Advantages of this module over other similar modules 150 151=over 4 152 153=item * Safety and Speed 154 155This module is safe yet fast. Alternatives are either fast but can cause segfaults or lose the mapping when not used correctly, or are safe but rather slow. File::Map is as fast as a normal string yet safe. 156 157=item * Simplicity 158 159It offers a simple interface targeted at common usage patterns 160 161=over 4 162 163=item * Files are mapped into a variable that can be read just like any other variable, and it can be written to using standard Perl techniques such as regexps and C<substr>. 164 165=item * Files can be mapped using a set of simple functions. There is no need to know weird constants or the order of 6 arguments. 166 167=item * It will automatically unmap the file when the scalar gets destroyed. This works correctly even in multi-threaded programs. 168 169=back 170 171=item * Portability 172 173File::Map supports Unix and Windows. 174 175=item * Thread synchronization 176 177It has built-in support for thread synchronization. 178 179=back 180 181=head1 FUNCTIONS 182 183=head2 Mapping 184 185The following functions for mapping a variable are available for exportation. Note that all of these functions throw exceptions on errors, unless noted otherwise. 186 187=head3 map_handle $lvalue, $filehandle, $mode = '<', $offset = 0, $length = -s(*handle) - $offset 188 189Use a filehandle to map into an lvalue. $filehandle should be a scalar filehandle. $mode uses the same format as C<open> does (it currently accepts C<< < >>, C<< +< >>, C<< > >> and C<< +> >>). $offset and $length are byte positions in the file, and default to mapping the whole file. 190 191=head3 * map_file $lvalue, $filename, $mode = '<', $offset = 0, $length = -s($filename) - $offset 192 193Open a file and map it into an lvalue. Other than $filename, all arguments work as in map_handle. 194 195=head3 * map_anonymous $lvalue, $length, $type 196 197Map an anonymous piece of memory. $type can be either C<'shared'>, in which case it will be shared with child processes, or C<'private'>, which won't be shared. 198 199=head3 * sys_map $lvalue, $length, $protection, $flags, $filehandle, $offset = 0 200 201Low level map operation. It accepts the same constants as mmap does (except its first argument obviously). If you don't know how mmap works you probably shouldn't be using this. 202 203=head3 * unmap $lvalue 204 205Unmap a variable. Note that normally this is not necessary as variables are unmapped automatically at destruction, but it is included for completeness. 206 207=head3 * remap $lvalue, $new_size 208 209Try to remap $lvalue to a new size. This call is linux specific and not supported on other systems. For a file backed mapping a file must be long enough to hold the new size, otherwise you can expect bus faults. For an anonymous map it must be private, shared maps can not be remapped. B<Use with caution>. 210 211=head2 Auxiliary 212 213=head3 * sync $lvalue, $synchronous = 1 214 215Flush changes made to the memory map back to disk. Mappings are always flushed when unmapped, so this is usually not necessary. If $synchronous is true and your operating system supports it, the flushing will be done synchronously. 216 217=head3 * pin $lvalue 218 219Disable paging for this map, thus locking it in physical memory. Depending on your operating system there may be limits on pinning. 220 221=head3 * unpin $lvalue 222 223Unlock the map from physical memory. 224 225=head3 * advise $lvalue, $advice 226 227Advise a certain memory usage pattern. This is not implemented on all operating systems, and may be a no-op. The following values for $advice are always accepted:. 228 229=over 2 230 231=item * normal 232 233Specifies that the application has no advice to give on its behavior with respect to the mapped variable. It is the default characteristic if no advice is given. 234 235=item * random 236 237Specifies that the application expects to access the mapped variable in a random order. 238 239=item * sequential 240 241Specifies that the application expects to access the mapped variable sequentially from start to end. 242 243=item * willneed 244 245Specifies that the application expects to access the mapped variable in the near future. 246 247=item * dontneed 248 249Specifies that the application expects that it will not access the mapped variable in the near future. 250 251=back 252 253On some systems there may be more values available, but this can not be relied on. Unknown values for $advice will cause a warning but are further ignored. 254 255=head3 * protect $lvalue, $mode 256 257Change the memory protection of the mapping. $mode takes the same format as C<open>, but also accepts sys_map style constants. 258 259=head2 Locking 260 261These locking functions provide locking for threads for the mapped region. The mapped region has an internal lock and condition variable. The condition variable functions(C<wait_until>, C<notify>, C<broadcast>) can only be used inside a locked block. If your perl has been compiled without thread support the condition functions will not be available. 262 263=head3 * lock_map $lvalue 264 265Lock $lvalue until the end of the scope. If your perl does not support threads, this will be a no-op. 266 267=head3 * wait_until { block } $lvalue 268 269Wait for block to become true. After every failed attempt, wait for a signal. It returns the value returned by the block. 270 271=head3 * notify $lvalue 272 273This will signal to one listener that the map is available. 274 275=head3 * broadcast $lvalue 276 277This will signal to all listeners that the map is available. 278 279=head2 Constants 280 281=over 4 282 283=item PROT_NONE, PROT_READ, PROT_WRITE, PROT_EXEC, MAP_ANONYMOUS, MAP_SHARED, MAP_PRIVATE, MAP_ANON, MAP_FILE 284 285These constants are used for sys_map. If you think you need them your mmap manpage will explain them, but in most cases you can skip sys_map altogether. 286 287=back 288 289=head1 EXPORTS 290 291All previously mentioned functions are available for exportation, but none are exported by default. Some functions may not be available on your OS or your version of perl as specified above. A number of tags are defined to make importation easier. 292 293=over 4 294 295=item * :map 296 297map_handle, map_file, map_anonymous, sys_map, unmap 298 299=item * :extra 300 301remap, sync, pin, unpin, advise, protect 302 303=item * :lock 304 305lock_map, wait_until, notify, broadcast 306 307=item * :constants 308 309PROT_NONE, PROT_READ, PROT_WRITE, PROT_EXEC, MAP_ANONYMOUS, MAP_SHARED, MAP_PRIVATE, MAP_ANON, MAP_FILE 310 311=item * :all 312 313All functions defined in this module. 314 315=back 316 317=head1 DIAGNOSTICS 318 319=head2 Exceptions 320 321=over 4 322 323=item * Could not <function name>: this variable is not memory mapped 324 325An attempt was made to C<sync>, C<remap>, C<unmap>, C<pin>, C<unpin>, C<advise> or C<lock_map> an unmapped variable. 326 327=item * Could not <function name>: <system error> 328 329Your OS didn't allow File::Map to do what you asked it to do for some reason. 330 331=item * Trying to <function_name> on an unlocked map 332 333You tried to C<wait_until>, C<notify> or C<broadcast> on an unlocked variable. 334 335=item * Zero length not allowed for anonymous map 336 337A zero length anonymous map is not possible (or in any way useful). 338 339=item * Can't remap a shared mapping 340 341An attempt was made to remap a mapping that is shared among different threads, this is not possible. 342 343=item * Window (<start>, <end>) is outside the file 344 345The offset and/or length you specified were invalid for this file. 346 347=item * Can't map fake filehandle 348 349The filehandle you provided is not real. This may mean it's a scalar string handle or a tied handle. 350 351=item * No such flag <flag_name> 352 353The flag given for map_anonymous isn't valid, it should either be C<shared> or C<private>. 354 355=back 356 357=head2 Warnings 358 359=over 4 360 361=item * Writing directly to a memory mapped file is not recommended 362 363Due to the way perl works internally, it's not possible to write a mapping implementation that allows direct assignment yet performs well. As a compromise, File::Map is capable of fixing up the mess if you do it nonetheless, but it will warn you that you're doing something you shouldn't. This warning is only given when C<use warnings 'substr'> is in effect. 364 365=item * Truncating new value to size of the memory map 366 367This warning is additional to the previous one, warning you that you're losing data. This warning is only given when C<use warnings 'substr'> is in effect. 368 369=item * Shouldn't mmap non-binary filehandle 370 371You tried to to map a filehandle that has some encoding layer. Encoding layers are not supported by File::Map. This warning is only given when C<use warnings 'layer'> is in effect. Note that this may become an exception in a future version. 372 373=item * Unknown advice '<advice>' 374 375You gave advise an advice it didn't know. This is either a typo or a portability issue. This warning is only given when C<use warnings 'portable'> is in effect. 376 377=item * Syncing a readonly map makes no sense 378 379C<sync> flushes changes to the map to the filesystem. This obviously is of little use when you can't change the map. This warning is only given when C<use warnings 'io'> is in effect. 380 381=item * Can't overwrite an empty map 382 383Overwriting an empty map is rather nonsensical, hence a warning is given when this is tried. This warning is only given when C<use warnings 'substr'> is in effect. 384 385=back 386 387=head1 DEPENDENCIES 388 389This module depends on perl 5.8, L<Sub::Exporter::Progressive> and L<PerlIO::Layers>. Perl 5.8.8 or higher is recommended because older versions can give spurious warnings. 390 391In perl versions before 5.11.5 many string functions including C<substr> are limited to L<32bit logic|http://rt.perl.org/rt3//Public/Bug/Display.html?id=72784>, even on 64bit architectures. Effectively this means you can't use them on strings bigger than 2GB. If you are working with such large files, it is strongly recommended to upgrade to 5.12. 392 393In perl versions before 5.17.5, there is an off-by-one bug in Perl's regexp engine, as explained L<here|http://rt.perl.org/rt3//Public/Bug/Display.html?id=73542>. If the length of the file is an exact multiple of the page size, some regexps can trigger a segmentation fault. 394 395=head1 PITFALLS 396 397=over 4 398 399=item * This module doesn't do any encoding or newline transformation for you, and will reject any filehandle with such features enabled as mapping it would return a different value than reading it normally. Most importantly this means that on Windows you have to remember to use the C<:raw> open mode or L<binmode> to make your filehandles binary before mapping them, as by default it would do C<crlf> transformation. See L<PerlIO> for more information on how that works. 400 401=item * You can map a C<:utf8> filehandle, but writing to it may be tricky. Hic sunt dracones. 402 403=item * You probably don't want to use C<E<gt>> as a mode. This does not give you reading permissions on many architectures, resulting in segmentation faults when trying to read a variable (confusingly, it will work on some others like x86). 404 405=back 406 407=head1 BUGS AND LIMITATIONS 408 409As any piece of software, bugs are likely to exist here. Bug reports are welcome. 410 411Please report any bugs or feature requests to C<bug-file-map at rt.cpan.org>, or through 412the web interface at L<http://rt.cpan.org/NoAuth/ReportBug.html?Queue=File-Map>. I will be notified, and then you'll 413automatically be notified of progress on your bug as I make changes. 414 415Unicode file mappings are known to be buggy on perl 5.8.7 and lower. 416 417=head1 SEE ALSO 418 419=over 4 420 421=item * L<Sys::Mmap>, the original Perl mmap module 422 423=item * L<mmap(2)>, your mmap man page 424 425=item * L<Win32::MMF> 426 427=item * CreateFileMapping at MSDN: L<http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx> 428 429=back 430 431=head1 AUTHOR 432 433Leon Timmermans <fawaka@gmail.com> 434 435=head1 COPYRIGHT AND LICENSE 436 437This software is copyright (c) 2008 by Leon Timmermans. 438 439This is free software; you can redistribute it and/or modify it under 440the same terms as the Perl 5 programming language system itself. 441 442=cut 443