1 // Written in the D programming language.
2
3 /**
4 * Read and write memory mapped files.
5 * Copyright: Copyright Digital Mars 2004 - 2009.
6 * License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0).
7 * Authors: $(HTTP digitalmars.com, Walter Bright),
8 * Matthew Wilson
9 * Source: $(PHOBOSSRC std/_mmfile.d)
10 *
11 * $(SCRIPT inhibitQuickIndex = 1;)
12 */
13 /* Copyright Digital Mars 2004 - 2009.
14 * Distributed under the Boost Software License, Version 1.0.
15 * (See accompanying file LICENSE_1_0.txt or copy at
16 * http://www.boost.org/LICENSE_1_0.txt)
17 */
18 module std.mmfile;
19
20 import core.stdc.errno;
21 import core.stdc.stdio;
22 import core.stdc.stdlib;
23 import std.conv, std.exception, std.stdio;
24 import std.file;
25 import std.path;
26 import std.string;
27
28 import std.internal.cstring;
29
30 //debug = MMFILE;
31
version(Windows)32 version (Windows)
33 {
34 import core.sys.windows.windows;
35 import std.utf;
36 import std.windows.syserror;
37 }
version(Posix)38 else version (Posix)
39 {
40 import core.sys.posix.fcntl;
41 import core.sys.posix.sys.mman;
42 import core.sys.posix.sys.stat;
43 import core.sys.posix.unistd;
44 }
45 else
46 {
47 static assert(0);
48 }
49
50 /**
51 * MmFile objects control the memory mapped file resource.
52 */
53 class MmFile
54 {
55 /**
56 * The mode the memory mapped file is opened with.
57 */
58 enum Mode
59 {
60 read, /// Read existing file
61 readWriteNew, /// Delete existing file, write new file
62 readWrite, /// Read/Write existing file, create if not existing
63 readCopyOnWrite, /// Read/Write existing file, copy on write
64 }
65
66 /**
67 * Open memory mapped file filename for reading.
68 * File is closed when the object instance is deleted.
69 * Throws:
70 * std.file.FileException
71 */
this(string filename)72 this(string filename)
73 {
74 this(filename, Mode.read, 0, null);
75 }
76
77 version (linux) this(File file, Mode mode = Mode.read, ulong size = 0,
78 void* address = null, size_t window = 0)
79 {
80 // Save a copy of the File to make sure the fd stays open.
81 this.file = file;
82 this(file.fileno, mode, size, address, window);
83 }
84
version(linux)85 version (linux) private this(int fildes, Mode mode, ulong size,
86 void* address, size_t window)
87 {
88 int oflag;
89 int fmode;
90
91 switch (mode)
92 {
93 case Mode.read:
94 flags = MAP_SHARED;
95 prot = PROT_READ;
96 oflag = O_RDONLY;
97 fmode = 0;
98 break;
99
100 case Mode.readWriteNew:
101 assert(size != 0);
102 flags = MAP_SHARED;
103 prot = PROT_READ | PROT_WRITE;
104 oflag = O_CREAT | O_RDWR | O_TRUNC;
105 fmode = octal!660;
106 break;
107
108 case Mode.readWrite:
109 flags = MAP_SHARED;
110 prot = PROT_READ | PROT_WRITE;
111 oflag = O_CREAT | O_RDWR;
112 fmode = octal!660;
113 break;
114
115 case Mode.readCopyOnWrite:
116 flags = MAP_PRIVATE;
117 prot = PROT_READ | PROT_WRITE;
118 oflag = O_RDWR;
119 fmode = 0;
120 break;
121
122 default:
123 assert(0);
124 }
125
126 fd = fildes;
127
128 // Adjust size
129 stat_t statbuf = void;
130 errnoEnforce(fstat(fd, &statbuf) == 0);
131 if (prot & PROT_WRITE && size > statbuf.st_size)
132 {
133 // Need to make the file size bytes big
134 lseek(fd, cast(off_t)(size - 1), SEEK_SET);
135 char c = 0;
136 core.sys.posix.unistd.write(fd, &c, 1);
137 }
138 else if (prot & PROT_READ && size == 0)
139 size = statbuf.st_size;
140 this.size = size;
141
142 // Map the file into memory!
143 size_t initial_map = (window && 2*window<size)
144 ? 2*window : cast(size_t) size;
145 auto p = mmap(address, initial_map, prot, flags, fd, 0);
146 if (p == MAP_FAILED)
147 {
148 errnoEnforce(false, "Could not map file into memory");
149 }
150 data = p[0 .. initial_map];
151 }
152
153 /**
154 * Open memory mapped file filename in mode.
155 * File is closed when the object instance is deleted.
156 * Params:
157 * filename = name of the file.
158 * If null, an anonymous file mapping is created.
159 * mode = access mode defined above.
160 * size = the size of the file. If 0, it is taken to be the
161 * size of the existing file.
162 * address = the preferred address to map the file to,
163 * although the system is not required to honor it.
164 * If null, the system selects the most convenient address.
165 * window = preferred block size of the amount of data to map at one time
166 * with 0 meaning map the entire file. The window size must be a
167 * multiple of the memory allocation page size.
168 * Throws:
169 * std.file.FileException
170 */
171 this(string filename, Mode mode, ulong size, void* address,
172 size_t window = 0)
173 {
174 this.filename = filename;
175 this.mMode = mode;
176 this.window = window;
177 this.address = address;
178
version(Windows)179 version (Windows)
180 {
181 void* p;
182 uint dwDesiredAccess2;
183 uint dwShareMode;
184 uint dwCreationDisposition;
185 uint flProtect;
186
187 switch (mode)
188 {
189 case Mode.read:
190 dwDesiredAccess2 = GENERIC_READ;
191 dwShareMode = FILE_SHARE_READ;
192 dwCreationDisposition = OPEN_EXISTING;
193 flProtect = PAGE_READONLY;
194 dwDesiredAccess = FILE_MAP_READ;
195 break;
196
197 case Mode.readWriteNew:
198 assert(size != 0);
199 dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
200 dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
201 dwCreationDisposition = CREATE_ALWAYS;
202 flProtect = PAGE_READWRITE;
203 dwDesiredAccess = FILE_MAP_WRITE;
204 break;
205
206 case Mode.readWrite:
207 dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
208 dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
209 dwCreationDisposition = OPEN_ALWAYS;
210 flProtect = PAGE_READWRITE;
211 dwDesiredAccess = FILE_MAP_WRITE;
212 break;
213
214 case Mode.readCopyOnWrite:
215 dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
216 dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
217 dwCreationDisposition = OPEN_EXISTING;
218 flProtect = PAGE_WRITECOPY;
219 dwDesiredAccess = FILE_MAP_COPY;
220 break;
221
222 default:
223 assert(0);
224 }
225
226 if (filename != null)
227 {
228 hFile = CreateFileW(filename.tempCStringW(),
229 dwDesiredAccess2,
230 dwShareMode,
231 null,
232 dwCreationDisposition,
233 FILE_ATTRIBUTE_NORMAL,
234 cast(HANDLE) null);
235 wenforce(hFile != INVALID_HANDLE_VALUE, "CreateFileW");
236 }
237 else
238 hFile = INVALID_HANDLE_VALUE;
239
240 scope(failure)
241 {
242 if (hFile != INVALID_HANDLE_VALUE)
243 {
244 CloseHandle(hFile);
245 hFile = INVALID_HANDLE_VALUE;
246 }
247 }
248
249 int hi = cast(int)(size >> 32);
250 hFileMap = CreateFileMappingW(hFile, null, flProtect,
251 hi, cast(uint) size, null);
252 wenforce(hFileMap, "CreateFileMapping");
253 scope(failure)
254 {
255 CloseHandle(hFileMap);
256 hFileMap = null;
257 }
258
259 if (size == 0 && filename != null)
260 {
261 uint sizehi;
262 uint sizelow = GetFileSize(hFile, &sizehi);
263 wenforce(sizelow != INVALID_FILE_SIZE || GetLastError() != ERROR_SUCCESS,
264 "GetFileSize");
265 size = (cast(ulong) sizehi << 32) + sizelow;
266 }
267 this.size = size;
268
269 size_t initial_map = (window && 2*window<size)
270 ? 2*window : cast(size_t) size;
271 p = MapViewOfFileEx(hFileMap, dwDesiredAccess, 0, 0,
272 initial_map, address);
273 wenforce(p, "MapViewOfFileEx");
274 data = p[0 .. initial_map];
275
276 debug (MMFILE) printf("MmFile.this(): p = %p, size = %d\n", p, size);
277 }
version(Posix)278 else version (Posix)
279 {
280 void* p;
281 int oflag;
282 int fmode;
283
284 switch (mode)
285 {
286 case Mode.read:
287 flags = MAP_SHARED;
288 prot = PROT_READ;
289 oflag = O_RDONLY;
290 fmode = 0;
291 break;
292
293 case Mode.readWriteNew:
294 assert(size != 0);
295 flags = MAP_SHARED;
296 prot = PROT_READ | PROT_WRITE;
297 oflag = O_CREAT | O_RDWR | O_TRUNC;
298 fmode = octal!660;
299 break;
300
301 case Mode.readWrite:
302 flags = MAP_SHARED;
303 prot = PROT_READ | PROT_WRITE;
304 oflag = O_CREAT | O_RDWR;
305 fmode = octal!660;
306 break;
307
308 case Mode.readCopyOnWrite:
309 flags = MAP_PRIVATE;
310 prot = PROT_READ | PROT_WRITE;
311 oflag = O_RDWR;
312 fmode = 0;
313 break;
314
315 default:
316 assert(0);
317 }
318
319 if (filename.length)
320 {
321 fd = .open(filename.tempCString(), oflag, fmode);
322 errnoEnforce(fd != -1, "Could not open file "~filename);
323
324 stat_t statbuf;
325 if (fstat(fd, &statbuf))
326 {
327 //printf("\tfstat error, errno = %d\n", errno);
328 .close(fd);
329 fd = -1;
330 errnoEnforce(false, "Could not stat file "~filename);
331 }
332
333 if (prot & PROT_WRITE && size > statbuf.st_size)
334 {
335 // Need to make the file size bytes big
336 .lseek(fd, cast(off_t)(size - 1), SEEK_SET);
337 char c = 0;
338 core.sys.posix.unistd.write(fd, &c, 1);
339 }
340 else if (prot & PROT_READ && size == 0)
341 size = statbuf.st_size;
342 }
343 else
344 {
345 fd = -1;
346 version (CRuntime_Glibc) import core.sys.linux.sys.mman : MAP_ANON;
347 flags |= MAP_ANON;
348 }
349 this.size = size;
350 size_t initial_map = (window && 2*window<size)
351 ? 2*window : cast(size_t) size;
352 p = mmap(address, initial_map, prot, flags, fd, 0);
353 if (p == MAP_FAILED)
354 {
355 if (fd != -1)
356 {
357 .close(fd);
358 fd = -1;
359 }
360 errnoEnforce(false, "Could not map file "~filename);
361 }
362
363 data = p[0 .. initial_map];
364 }
365 else
366 {
367 static assert(0);
368 }
369 }
370
371 /**
372 * Flushes pending output and closes the memory mapped file.
373 */
~this()374 ~this()
375 {
376 debug (MMFILE) printf("MmFile.~this()\n");
377 unmap();
378 data = null;
379 version (Windows)
380 {
381 wenforce(hFileMap == null || CloseHandle(hFileMap) == TRUE,
382 "Could not close file handle");
383 hFileMap = null;
384
385 wenforce(!hFile || hFile == INVALID_HANDLE_VALUE
386 || CloseHandle(hFile) == TRUE,
387 "Could not close handle");
388 hFile = INVALID_HANDLE_VALUE;
389 }
390 else version (Posix)
391 {
392 version (linux)
393 {
394 if (file !is File.init)
395 {
396 // The File destructor will close the file,
397 // if it is the only remaining reference.
398 return;
399 }
400 }
401 errnoEnforce(fd == -1 || fd <= 2
402 || .close(fd) != -1,
403 "Could not close handle");
404 fd = -1;
405 }
406 else
407 {
408 static assert(0);
409 }
410 }
411
412 /* Flush any pending output.
413 */
flush()414 void flush()
415 {
416 debug (MMFILE) printf("MmFile.flush()\n");
417 version (Windows)
418 {
419 FlushViewOfFile(data.ptr, data.length);
420 }
421 else version (Posix)
422 {
423 int i;
424 i = msync(cast(void*) data, data.length, MS_SYNC); // sys/mman.h
425 errnoEnforce(i == 0, "msync failed");
426 }
427 else
428 {
429 static assert(0);
430 }
431 }
432
433 /**
434 * Gives size in bytes of the memory mapped file.
435 */
length()436 @property ulong length() const
437 {
438 debug (MMFILE) printf("MmFile.length()\n");
439 return size;
440 }
441
442 /**
443 * Read-only property returning the file mode.
444 */
mode()445 Mode mode()
446 {
447 debug (MMFILE) printf("MmFile.mode()\n");
448 return mMode;
449 }
450
451 /**
452 * Returns entire file contents as an array.
453 */
opSlice()454 void[] opSlice()
455 {
456 debug (MMFILE) printf("MmFile.opSlice()\n");
457 return opSlice(0,size);
458 }
459
460 /**
461 * Returns slice of file contents as an array.
462 */
opSlice(ulong i1,ulong i2)463 void[] opSlice(ulong i1, ulong i2)
464 {
465 debug (MMFILE) printf("MmFile.opSlice(%lld, %lld)\n", i1, i2);
466 ensureMapped(i1,i2);
467 size_t off1 = cast(size_t)(i1-start);
468 size_t off2 = cast(size_t)(i2-start);
469 return data[off1 .. off2];
470 }
471
472 /**
473 * Returns byte at index i in file.
474 */
opIndex(ulong i)475 ubyte opIndex(ulong i)
476 {
477 debug (MMFILE) printf("MmFile.opIndex(%lld)\n", i);
478 ensureMapped(i);
479 size_t off = cast(size_t)(i-start);
480 return (cast(ubyte[]) data)[off];
481 }
482
483 /**
484 * Sets and returns byte at index i in file to value.
485 */
opIndexAssign(ubyte value,ulong i)486 ubyte opIndexAssign(ubyte value, ulong i)
487 {
488 debug (MMFILE) printf("MmFile.opIndex(%lld, %d)\n", i, value);
489 ensureMapped(i);
490 size_t off = cast(size_t)(i-start);
491 return (cast(ubyte[]) data)[off] = value;
492 }
493
494
495 // return true if the given position is currently mapped
mapped(ulong i)496 private int mapped(ulong i)
497 {
498 debug (MMFILE) printf("MmFile.mapped(%lld, %lld, %d)\n", i,start,
499 data.length);
500 return i >= start && i < start+data.length;
501 }
502
503 // unmap the current range
unmap()504 private void unmap()
505 {
506 debug (MMFILE) printf("MmFile.unmap()\n");
507 version (Windows)
508 {
509 wenforce(!data.ptr || UnmapViewOfFile(data.ptr) != FALSE, "UnmapViewOfFile");
510 }
511 else
512 {
513 errnoEnforce(!data.ptr || munmap(cast(void*) data, data.length) == 0,
514 "munmap failed");
515 }
516 data = null;
517 }
518
519 // map range
map(ulong start,size_t len)520 private void map(ulong start, size_t len)
521 {
522 debug (MMFILE) printf("MmFile.map(%lld, %d)\n", start, len);
523 void* p;
524 if (start+len > size)
525 len = cast(size_t)(size-start);
526 version (Windows)
527 {
528 uint hi = cast(uint)(start >> 32);
529 p = MapViewOfFileEx(hFileMap, dwDesiredAccess, hi, cast(uint) start, len, address);
530 wenforce(p, "MapViewOfFileEx");
531 }
532 else
533 {
534 p = mmap(address, len, prot, flags, fd, cast(off_t) start);
535 errnoEnforce(p != MAP_FAILED);
536 }
537 data = p[0 .. len];
538 this.start = start;
539 }
540
541 // ensure a given position is mapped
ensureMapped(ulong i)542 private void ensureMapped(ulong i)
543 {
544 debug (MMFILE) printf("MmFile.ensureMapped(%lld)\n", i);
545 if (!mapped(i))
546 {
547 unmap();
548 if (window == 0)
549 {
550 map(0,cast(size_t) size);
551 }
552 else
553 {
554 ulong block = i/window;
555 if (block == 0)
556 map(0,2*window);
557 else
558 map(window*(block-1),3*window);
559 }
560 }
561 }
562
563 // ensure a given range is mapped
ensureMapped(ulong i,ulong j)564 private void ensureMapped(ulong i, ulong j)
565 {
566 debug (MMFILE) printf("MmFile.ensureMapped(%lld, %lld)\n", i, j);
567 if (!mapped(i) || !mapped(j-1))
568 {
569 unmap();
570 if (window == 0)
571 {
572 map(0,cast(size_t) size);
573 }
574 else
575 {
576 ulong iblock = i/window;
577 ulong jblock = (j-1)/window;
578 if (iblock == 0)
579 {
580 map(0,cast(size_t)(window*(jblock+2)));
581 }
582 else
583 {
584 map(window*(iblock-1),cast(size_t)(window*(jblock-iblock+3)));
585 }
586 }
587 }
588 }
589
590 private:
591 string filename;
592 void[] data;
593 ulong start;
594 size_t window;
595 ulong size;
596 Mode mMode;
597 void* address;
version(linux)598 version (linux) File file;
599
600 version (Windows)
601 {
602 HANDLE hFile = INVALID_HANDLE_VALUE;
603 HANDLE hFileMap = null;
604 uint dwDesiredAccess;
605 }
version(Posix)606 else version (Posix)
607 {
608 int fd;
609 int prot;
610 int flags;
611 int fmode;
612 }
613 else
614 {
615 static assert(0);
616 }
617
618 // Report error, where errno gives the error number
619 // void errNo()
620 // {
621 // version (Windows)
622 // {
623 // throw new FileException(filename, GetLastError());
624 // }
625 // else version (linux)
626 // {
627 // throw new FileException(filename, errno);
628 // }
629 // else
630 // {
631 // static assert(0);
632 // }
633 // }
634 }
635
636 @system unittest
637 {
638 import core.memory : GC;
639 import std.file : deleteme;
640
641 const size_t K = 1024;
642 size_t win = 64*K; // assume the page size is 64K
version(Windows)643 version (Windows)
644 {
645 /+ these aren't defined in core.sys.windows.windows so let's use default
646 SYSTEM_INFO sysinfo;
647 GetSystemInfo(&sysinfo);
648 win = sysinfo.dwAllocationGranularity;
649 +/
650 }
version(linux)651 else version (linux)
652 {
653 // getpagesize() is not defined in the unix D headers so use the guess
654 }
655 string test_file = std.file.deleteme ~ "-testing.txt";
656 MmFile mf = new MmFile(test_file,MmFile.Mode.readWriteNew,
657 100*K,null,win);
658 ubyte[] str = cast(ubyte[])"1234567890";
659 ubyte[] data = cast(ubyte[]) mf[0 .. 10];
660 data[] = str[];
661 assert( mf[0 .. 10] == str );
662 data = cast(ubyte[]) mf[50 .. 60];
663 data[] = str[];
664 assert( mf[50 .. 60] == str );
665 ubyte[] data2 = cast(ubyte[]) mf[20*K .. 60*K];
666 assert( data2.length == 40*K );
667 assert( data2[$-1] == 0 );
668 mf[100*K-1] = cast(ubyte)'b';
669 data2 = cast(ubyte[]) mf[21*K .. 100*K];
670 assert( data2.length == 79*K );
671 assert( data2[$-1] == 'b' );
672
673 destroy(mf);
674 GC.free(&mf);
675
676 std.file.remove(test_file);
677 // Create anonymous mapping
678 auto test = new MmFile(null, MmFile.Mode.readWriteNew, 1024*1024, null);
679 }
680
version(linux)681 version (linux)
682 @system unittest // Issue 14868
683 {
684 import std.file : deleteme;
685 import std.typecons : scoped;
686
687 // Test retaining ownership of File/fd
688
689 auto fn = std.file.deleteme ~ "-testing.txt";
690 scope(exit) std.file.remove(fn);
691 File(fn, "wb").writeln("Testing!");
692 scoped!MmFile(File(fn));
693
694 // Test that unique ownership of File actually leads to the fd being closed
695
696 auto f = File(fn);
697 auto fd = f.fileno;
698 {
699 auto mf = scoped!MmFile(f);
700 f = File.init;
701 }
702 assert(.close(fd) == -1);
703 }
704
705 @system unittest // Issue 14994, 14995
706 {
707 import std.file : deleteme;
708 import std.typecons : scoped;
709
710 // Zero-length map may or may not be valid on OSX and NetBSD
711 version (OSX)
712 import std.exception : verifyThrown = collectException;
713 version (NetBSD)
714 import std.exception : verifyThrown = collectException;
715 else
716 import std.exception : verifyThrown = assertThrown;
717
718 auto fn = std.file.deleteme ~ "-testing.txt";
719 scope(exit) std.file.remove(fn);
720 verifyThrown(scoped!MmFile(fn, MmFile.Mode.readWrite, 0, null));
721 }
722