1 /* <!-- copyright */
2 /*
3 * aria2 - The high speed download utility
4 *
5 * Copyright (C) 2006 Tatsuhiro Tsujikawa
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 *
21 * In addition, as a special exception, the copyright holders give
22 * permission to link the code of portions of this program with the
23 * OpenSSL library under certain conditions as described in each
24 * individual source file, and distribute linked combinations
25 * including the two.
26 * You must obey the GNU General Public License in all respects
27 * for all of the code used other than OpenSSL. If you modify
28 * file(s) with this exception, you may extend this exception to your
29 * version of the file(s), but you are not obligated to do so. If you
30 * do not wish to do so, delete this exception statement from your
31 * version. If you delete this exception statement from all source
32 * files in the program, then also delete it here.
33 */
34 /* copyright --> */
35 #include "AbstractDiskWriter.h"
36
37 #include <unistd.h>
38 #ifdef HAVE_MMAP
39 # include <sys/mman.h>
40 #endif // HAVE_MMAP
41 #include <fcntl.h>
42
43 #include <cerrno>
44 #include <cstring>
45 #include <cassert>
46
47 #include "File.h"
48 #include "util.h"
49 #include "message.h"
50 #include "DlAbortEx.h"
51 #include "a2io.h"
52 #include "fmt.h"
53 #include "DownloadFailureException.h"
54 #include "error_code.h"
55 #include "LogFactory.h"
56
57 namespace aria2 {
58
AbstractDiskWriter(const std::string & filename)59 AbstractDiskWriter::AbstractDiskWriter(const std::string& filename)
60 : filename_(filename),
61 fd_(A2_BAD_FD),
62 #ifdef __MINGW32__
63 mapView_(0),
64 #else // !__MINGW32__
65 #endif // !__MINGW32__
66 readOnly_(false),
67 enableMmap_(false),
68 mapaddr_(nullptr),
69 maplen_(0)
70
71 {
72 }
73
~AbstractDiskWriter()74 AbstractDiskWriter::~AbstractDiskWriter() { closeFile(); }
75
76 namespace {
77 // Returns error code depending on the platform. For MinGW32, return
78 // the value of GetLastError(). Otherwise, return errno.
fileError()79 int fileError()
80 {
81 #ifdef __MINGW32__
82 return GetLastError();
83 #else // !__MINGW32__
84 return errno;
85 #endif // !__MINGW32__
86 }
87 } // namespace
88
89 namespace {
90 // Formats error message for error code errNum. For MinGW32, errNum is
91 // assumed to be the return value of GetLastError(). Otherwise, it is
92 // errno.
fileStrerror(int errNum)93 std::string fileStrerror(int errNum)
94 {
95 #ifdef __MINGW32__
96 auto msg = util::formatLastError(errNum);
97 if (msg.empty()) {
98 char buf[256];
99 snprintf(buf, sizeof(buf), "File I/O error %x", errNum);
100 return buf;
101 }
102 return msg;
103 #else // !__MINGW32__
104 return util::safeStrerror(errNum);
105 #endif // !__MINGW32__
106 }
107 } // namespace
108
openFile(int64_t totalLength)109 void AbstractDiskWriter::openFile(int64_t totalLength)
110 {
111 try {
112 openExistingFile(totalLength);
113 }
114 catch (RecoverableException& e) {
115 if (
116 #ifdef __MINGW32__
117 e.getErrNum() == ERROR_FILE_NOT_FOUND ||
118 e.getErrNum() == ERROR_PATH_NOT_FOUND
119 #else // !__MINGW32__
120 e.getErrNum() == ENOENT
121 #endif // !__MINGW32__
122 ) {
123 initAndOpenFile(totalLength);
124 }
125 else {
126 throw;
127 }
128 }
129 }
130
closeFile()131 void AbstractDiskWriter::closeFile()
132 {
133 #if defined(HAVE_MMAP) || defined(__MINGW32__)
134 if (mapaddr_) {
135 int errNum = 0;
136 # ifdef __MINGW32__
137 if (!UnmapViewOfFile(mapaddr_)) {
138 errNum = GetLastError();
139 }
140 CloseHandle(mapView_);
141 mapView_ = INVALID_HANDLE_VALUE;
142 # else // !__MINGW32__
143 if (munmap(mapaddr_, maplen_) == -1) {
144 errNum = errno;
145 }
146 # endif // !__MINGW32__
147 if (errNum != 0) {
148 int errNum = fileError();
149 A2_LOG_ERROR(fmt("Unmapping file %s failed: %s", filename_.c_str(),
150 fileStrerror(errNum).c_str()));
151 }
152 else {
153 A2_LOG_INFO(fmt("Unmapping file %s succeeded", filename_.c_str()));
154 }
155 mapaddr_ = nullptr;
156 maplen_ = 0;
157 }
158 #endif // HAVE_MMAP || defined __MINGW32__
159 if (fd_ != A2_BAD_FD) {
160 #ifdef __MINGW32__
161 CloseHandle(fd_);
162 #else // !__MINGW32__
163 close(fd_);
164 #endif // !__MINGW32__
165 fd_ = A2_BAD_FD;
166 }
167 }
168
169 namespace {
170 #ifdef __MINGW32__
openFileWithFlags(const std::string & filename,int flags,error_code::Value errCode)171 HANDLE openFileWithFlags(const std::string& filename, int flags,
172 error_code::Value errCode)
173 {
174 HANDLE hn;
175 DWORD desiredAccess = 0;
176 DWORD sharedMode = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE;
177 DWORD creationDisp = 0;
178
179 if (flags & O_RDWR) {
180 desiredAccess = GENERIC_READ | GENERIC_WRITE;
181 }
182 else if (flags & O_WRONLY) {
183 desiredAccess = GENERIC_WRITE;
184 }
185 else {
186 desiredAccess = GENERIC_READ;
187 }
188 if (flags & O_CREAT) {
189 if (flags & O_TRUNC) {
190 creationDisp |= CREATE_ALWAYS;
191 }
192 else {
193 creationDisp |= CREATE_NEW;
194 }
195 }
196 else {
197 creationDisp |= OPEN_EXISTING;
198 }
199 hn = CreateFileW(utf8ToWChar(filename).c_str(), desiredAccess, sharedMode,
200 /* lpSecurityAttributes */ 0, creationDisp,
201 FILE_ATTRIBUTE_NORMAL, /* hTemplateFile */ 0);
202 if (hn == INVALID_HANDLE_VALUE) {
203 int errNum = GetLastError();
204 throw DL_ABORT_EX3(
205 errNum,
206 fmt(EX_FILE_OPEN, filename.c_str(), fileStrerror(errNum).c_str()),
207 errCode);
208 }
209 return hn;
210 }
211 #else // !__MINGW32__
212 int openFileWithFlags(const std::string& filename, int flags,
213 error_code::Value errCode)
214 {
215 int fd;
216 while ((fd = a2open(utf8ToWChar(filename).c_str(), flags, OPEN_MODE)) == -1 &&
217 errno == EINTR)
218 ;
219 if (fd < 0) {
220 int errNum = errno;
221 throw DL_ABORT_EX3(
222 errNum,
223 fmt(EX_FILE_OPEN, filename.c_str(), util::safeStrerror(errNum).c_str()),
224 errCode);
225 }
226 util::make_fd_cloexec(fd);
227 # if defined(__APPLE__) && defined(__MACH__)
228 // This may reduce memory consumption on Mac OS X.
229 fcntl(fd, F_NOCACHE, 1);
230 # endif // __APPLE__ && __MACH__
231 return fd;
232 }
233 #endif // !__MINGW32__
234 } // namespace
235
openExistingFile(int64_t totalLength)236 void AbstractDiskWriter::openExistingFile(int64_t totalLength)
237 {
238 int flags = O_BINARY;
239 if (readOnly_) {
240 flags |= O_RDONLY;
241 }
242 else {
243 flags |= O_RDWR;
244 }
245 fd_ = openFileWithFlags(filename_, flags, error_code::FILE_OPEN_ERROR);
246 }
247
createFile(int addFlags)248 void AbstractDiskWriter::createFile(int addFlags)
249 {
250 assert(!filename_.empty());
251 util::mkdirs(File(filename_).getDirname());
252 fd_ = openFileWithFlags(filename_,
253 O_CREAT | O_RDWR | O_TRUNC | O_BINARY | addFlags,
254 error_code::FILE_CREATE_ERROR);
255 }
256
writeDataInternal(const unsigned char * data,size_t len,int64_t offset)257 ssize_t AbstractDiskWriter::writeDataInternal(const unsigned char* data,
258 size_t len, int64_t offset)
259 {
260 if (mapaddr_) {
261 memcpy(mapaddr_ + offset, data, len);
262 return len;
263 }
264 else {
265 ssize_t writtenLength = 0;
266 seek(offset);
267 while ((size_t)writtenLength < len) {
268 #ifdef __MINGW32__
269 DWORD nwrite;
270 if (WriteFile(fd_, data + writtenLength, len - writtenLength, &nwrite,
271 0)) {
272 writtenLength += nwrite;
273 }
274 else {
275 return -1;
276 }
277 #else // !__MINGW32__
278 ssize_t ret = 0;
279 while ((ret = write(fd_, data + writtenLength, len - writtenLength)) ==
280 -1 &&
281 errno == EINTR)
282 ;
283 if (ret == -1) {
284 return -1;
285 }
286 writtenLength += ret;
287 #endif // !__MINGW32__
288 }
289 return writtenLength;
290 }
291 }
292
readDataInternal(unsigned char * data,size_t len,int64_t offset)293 ssize_t AbstractDiskWriter::readDataInternal(unsigned char* data, size_t len,
294 int64_t offset)
295 {
296 if (mapaddr_) {
297 if (offset >= maplen_) {
298 return 0;
299 }
300 auto readlen = std::min(maplen_ - offset, static_cast<int64_t>(len));
301 memcpy(data, mapaddr_ + offset, readlen);
302 return readlen;
303 }
304 else {
305 seek(offset);
306 #ifdef __MINGW32__
307 DWORD nread;
308 if (ReadFile(fd_, data, len, &nread, 0)) {
309 return nread;
310 }
311 else {
312 return -1;
313 }
314 #else // !__MINGW32__
315 ssize_t ret = 0;
316 while ((ret = read(fd_, data, len)) == -1 && errno == EINTR)
317 ;
318 return ret;
319 #endif // !__MINGW32__
320 }
321 }
322
seek(int64_t offset)323 void AbstractDiskWriter::seek(int64_t offset)
324 {
325 assert(offset >= 0);
326 #ifdef __MINGW32__
327 LARGE_INTEGER fileLength;
328 fileLength.QuadPart = offset;
329 if (SetFilePointerEx(fd_, fileLength, 0, FILE_BEGIN) == 0)
330 #else // !__MINGW32__
331 if (a2lseek(fd_, offset, SEEK_SET) == (a2_off_t)-1)
332 #endif // !__MINGW32__
333 {
334 int errNum = fileError();
335 throw DL_ABORT_EX2(
336 fmt(EX_FILE_SEEK, filename_.c_str(), fileStrerror(errNum).c_str()),
337 error_code::FILE_IO_ERROR);
338 }
339 }
340
ensureMmapWrite(size_t len,int64_t offset)341 void AbstractDiskWriter::ensureMmapWrite(size_t len, int64_t offset)
342 {
343 #if defined(HAVE_MMAP) || defined(__MINGW32__)
344 if (enableMmap_) {
345 if (mapaddr_) {
346 if (static_cast<int64_t>(len + offset) > maplen_) {
347 int errNum = 0;
348 # ifdef __MINGW32__
349 if (!UnmapViewOfFile(mapaddr_)) {
350 errNum = GetLastError();
351 }
352 CloseHandle(mapView_);
353 mapView_ = INVALID_HANDLE_VALUE;
354 # else // !__MINGW32__
355 if (munmap(mapaddr_, maplen_) == -1) {
356 errNum = errno;
357 }
358 # endif // !__MINGW32__
359 if (errNum != 0) {
360 A2_LOG_ERROR(fmt("Unmapping file %s failed: %s", filename_.c_str(),
361 fileStrerror(errNum).c_str()));
362 }
363 mapaddr_ = nullptr;
364 maplen_ = 0;
365 enableMmap_ = false;
366 }
367 }
368 else {
369 int64_t filesize = size();
370
371 if (filesize == 0) {
372 // mapping 0 length file is useless. Also munmap with size ==
373 // 0 will fail with EINVAL.
374 enableMmap_ = false;
375 return;
376 }
377
378 if (static_cast<uint64_t>(std::numeric_limits<size_t>::max()) <
379 static_cast<uint64_t>(filesize)) {
380 // filesize could overflow in 32bit OS with 64bit off_t type
381 // the filesize will be truncated if provided as a 32bit size_t
382 enableMmap_ = false;
383 return;
384 }
385
386 int errNum = 0;
387 if (static_cast<int64_t>(len + offset) <= filesize) {
388 # ifdef __MINGW32__
389 mapView_ = CreateFileMapping(fd_, 0, PAGE_READWRITE, filesize >> 32,
390 filesize & 0xffffffffu, 0);
391 if (mapView_) {
392 mapaddr_ = reinterpret_cast<unsigned char*>(
393 MapViewOfFile(mapView_, FILE_MAP_WRITE, 0, 0, 0));
394 if (!mapaddr_) {
395 errNum = GetLastError();
396 CloseHandle(mapView_);
397 mapView_ = INVALID_HANDLE_VALUE;
398 }
399 }
400 else {
401 errNum = GetLastError();
402 }
403 # else // !__MINGW32__
404 auto pa =
405 mmap(nullptr, filesize, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0);
406
407 if (pa == MAP_FAILED) {
408 errNum = errno;
409 }
410 else {
411 mapaddr_ = reinterpret_cast<unsigned char*>(pa);
412 }
413 # endif // !__MINGW32__
414 if (mapaddr_) {
415 A2_LOG_DEBUG(fmt("Mapping file %s succeeded, length=%" PRId64 "",
416 filename_.c_str(), static_cast<uint64_t>(filesize)));
417 maplen_ = filesize;
418 }
419 else {
420 A2_LOG_WARN(fmt("Mapping file %s failed: %s", filename_.c_str(),
421 fileStrerror(errNum).c_str()));
422 enableMmap_ = false;
423 }
424 }
425 }
426 }
427 #endif // HAVE_MMAP || __MINGW32__
428 }
429
430 namespace {
431 // Returns true if |errNum| indicates that disk is full.
isDiskFullError(int errNum)432 bool isDiskFullError(int errNum)
433 {
434 return
435 #ifdef __MINGW32__
436 errNum == ERROR_DISK_FULL || errNum == ERROR_HANDLE_DISK_FULL
437 #else // !__MINGW32__
438 errNum == ENOSPC
439 #endif // !__MINGW32__
440 ;
441 }
442 } // namespace
443
writeData(const unsigned char * data,size_t len,int64_t offset)444 void AbstractDiskWriter::writeData(const unsigned char* data, size_t len,
445 int64_t offset)
446 {
447 ensureMmapWrite(len, offset);
448 if (writeDataInternal(data, len, offset) < 0) {
449 int errNum = fileError();
450 // If the error indicates disk full situation, throw
451 // DownloadFailureException and abort download instantly.
452 if (isDiskFullError(errNum)) {
453 throw DOWNLOAD_FAILURE_EXCEPTION3(
454 errNum,
455 fmt(EX_FILE_WRITE, filename_.c_str(), fileStrerror(errNum).c_str()),
456 error_code::NOT_ENOUGH_DISK_SPACE);
457 }
458 else {
459 throw DL_ABORT_EX3(
460 errNum,
461 fmt(EX_FILE_WRITE, filename_.c_str(), fileStrerror(errNum).c_str()),
462 error_code::FILE_IO_ERROR);
463 }
464 }
465 }
466
readData(unsigned char * data,size_t len,int64_t offset)467 ssize_t AbstractDiskWriter::readData(unsigned char* data, size_t len,
468 int64_t offset)
469 {
470 ssize_t ret;
471 if ((ret = readDataInternal(data, len, offset)) < 0) {
472 int errNum = fileError();
473 throw DL_ABORT_EX3(
474 errNum,
475 fmt(EX_FILE_READ, filename_.c_str(), fileStrerror(errNum).c_str()),
476 error_code::FILE_IO_ERROR);
477 }
478 return ret;
479 }
480
truncate(int64_t length)481 void AbstractDiskWriter::truncate(int64_t length)
482 {
483 if (fd_ == A2_BAD_FD) {
484 throw DL_ABORT_EX("File not yet opened.");
485 }
486 #ifdef __MINGW32__
487 // Since mingw32's ftruncate cannot handle over 2GB files, we use
488 // SetEndOfFile instead.
489 seek(length);
490 if (SetEndOfFile(fd_) == 0)
491 #else // !__MINGW32__
492 if (a2ftruncate(fd_, length) == -1)
493 #endif // !__MINGW32__
494 {
495 int errNum = fileError();
496 throw DL_ABORT_EX2(
497 fmt("File truncation failed. cause: %s", fileStrerror(errNum).c_str()),
498 error_code::FILE_IO_ERROR);
499 }
500 }
501
allocate(int64_t offset,int64_t length,bool sparse)502 void AbstractDiskWriter::allocate(int64_t offset, int64_t length, bool sparse)
503 {
504 if (fd_ == A2_BAD_FD) {
505 throw DL_ABORT_EX("File not yet opened.");
506 }
507 if (sparse) {
508 #ifdef __MINGW32__
509 DWORD bytesReturned;
510 if (!DeviceIoControl(fd_, FSCTL_SET_SPARSE, 0, 0, 0, 0, &bytesReturned,
511 0)) {
512 A2_LOG_WARN(fmt("Making file sparse failed or pending: %s",
513 fileStrerror(GetLastError()).c_str()));
514 }
515 #endif // __MINGW32__
516 truncate(offset + length);
517 return;
518 }
519 #ifdef HAVE_SOME_FALLOCATE
520 # ifdef __MINGW32__
521 truncate(offset + length);
522 if (!SetFileValidData(fd_, offset + length)) {
523 auto errNum = fileError();
524 A2_LOG_WARN(fmt(
525 "File allocation (SetFileValidData) failed (cause: %s). File will be "
526 "allocated by filling zero, which blocks whole aria2 execution. Run "
527 "aria2 as an administrator or use a different file allocation method "
528 "(see --file-allocation).",
529 fileStrerror(errNum).c_str()));
530 }
531 # elif defined(__APPLE__) && defined(__MACH__)
532 const auto toalloc = offset + length - size();
533 fstore_t fstore = {F_ALLOCATECONTIG | F_ALLOCATEALL, F_PEOFPOSMODE, 0,
534 toalloc, 0};
535 if (fcntl(fd_, F_PREALLOCATE, &fstore) == -1) {
536 // Retry non-contig.
537 fstore.fst_flags = F_ALLOCATEALL;
538 if (fcntl(fd_, F_PREALLOCATE, &fstore) == -1) {
539 int err = errno;
540 throw DL_ABORT_EX3(
541 err,
542 fmt("fcntl(F_PREALLOCATE) of %" PRId64 " failed. cause: %s",
543 fstore.fst_length, util::safeStrerror(err).c_str()),
544 error_code::FILE_IO_ERROR);
545 }
546 }
547 // This forces the allocation on disk.
548 ftruncate(fd_, offset + length);
549 # elif HAVE_FALLOCATE
550 // For linux, we use fallocate to detect file system supports
551 // fallocate or not.
552 int r;
553 while ((r = fallocate(fd_, 0, offset, length)) == -1 && errno == EINTR)
554 ;
555 int errNum = errno;
556 if (r == -1) {
557 throw DL_ABORT_EX3(
558 errNum,
559 fmt("fallocate failed. cause: %s", util::safeStrerror(errNum).c_str()),
560 isDiskFullError(errNum) ? error_code::NOT_ENOUGH_DISK_SPACE
561 : error_code::FILE_IO_ERROR);
562 }
563 # elif HAVE_POSIX_FALLOCATE
564 int r = posix_fallocate(fd_, offset, length);
565 if (r != 0) {
566 throw DL_ABORT_EX3(
567 r,
568 fmt("posix_fallocate failed. cause: %s", util::safeStrerror(r).c_str()),
569 isDiskFullError(r) ? error_code::NOT_ENOUGH_DISK_SPACE
570 : error_code::FILE_IO_ERROR);
571 }
572 # else
573 # error "no *_fallocate function available."
574 # endif
575 #endif // HAVE_SOME_FALLOCATE
576 }
577
size()578 int64_t AbstractDiskWriter::size() { return File(filename_).size(); }
579
enableReadOnly()580 void AbstractDiskWriter::enableReadOnly() { readOnly_ = true; }
581
disableReadOnly()582 void AbstractDiskWriter::disableReadOnly() { readOnly_ = false; }
583
enableMmap()584 void AbstractDiskWriter::enableMmap() { enableMmap_ = true; }
585
dropCache(int64_t len,int64_t offset)586 void AbstractDiskWriter::dropCache(int64_t len, int64_t offset)
587 {
588 #ifdef HAVE_POSIX_FADVISE
589 posix_fadvise(fd_, offset, len, POSIX_FADV_DONTNEED);
590 #endif // HAVE_POSIX_FADVISE
591 }
592
flushOSBuffers()593 void AbstractDiskWriter::flushOSBuffers()
594 {
595 if (fd_ == A2_BAD_FD) {
596 return;
597 }
598 #ifdef __MINGW32__
599 FlushFileBuffers(fd_);
600 #else // !__MINGW32__
601 fsync(fd_);
602 #endif // __MINGW32__
603 }
604
605 } // namespace aria2
606