1
2 //#define LOCAL_DEBUG
3 #include "debug.h"
4
5 #include "config.h"
6 #include "fileitem.h"
7 #include "header.h"
8 #include "acfg.h"
9 #include "acbuf.h"
10 #include "fileio.h"
11 #include "cleaner.h"
12 #include "filelocks.h"
13
14 #include <errno.h>
15 #include <algorithm>
16
17 #include <sys/stat.h>
18 #include <sys/types.h>
19 #include <sys/time.h>
20
21 using namespace std;
22
23 namespace acng
24 {
25 #define MAXTEMPDELAY acng::cfg::maxtempdelay // 27
26 mstring sReplDir("_altStore" SZPATHSEP);
27
28 static tFiGlobMap mapItems;
29 #ifndef MINIBUILD
30 static acmutex mapItemsMx;
31 #endif
32
GetHeaderUnlocked()33 header const & fileitem::GetHeaderUnlocked()
34 {
35 return m_head;
36 }
37
GetHttpMsg()38 string fileitem::GetHttpMsg()
39 {
40 setLockGuard;
41 if(m_head.frontLine.length()>9)
42 return m_head.frontLine.substr(9);
43 return m_head.frontLine;
44 }
45
fileitem()46 fileitem::fileitem() :
47 m_nIncommingCount(0),
48 m_nSizeSeen(0),
49 m_nRangeLimit(-1),
50 m_bCheckFreshness(true),
51 m_bHeadOnly(false),
52 m_bAllowStoreData(true),
53 m_nSizeChecked(0),
54 m_filefd(-1),
55 m_nDlRefsCount(0),
56 usercount(0),
57 m_status(FIST_FRESH),
58 m_nTimeDlStarted(0),
59 m_nTimeDlDone(END_OF_TIME),
60 m_globRef(mapItems.end())
61 {
62 }
63
~fileitem()64 fileitem::~fileitem()
65 {
66 //setLockGuard;
67 // m_head.clear();
68 Truncate2checkedSize();
69 checkforceclose(m_filefd);
70 }
71
IncDlRefCount()72 void fileitem::IncDlRefCount()
73 {
74 setLockGuard;
75 m_nDlRefsCount++;
76 }
77
DecDlRefCount(const string & sReason)78 void fileitem::DecDlRefCount(const string &sReason)
79 {
80 setLockGuard;
81
82 notifyAll();
83
84 m_nDlRefsCount--;
85 if(m_nDlRefsCount>0)
86 return; // someone will care...
87
88 // ... otherwise: the last downloader disappeared, needing to tell observers
89
90 if (m_status<FIST_COMPLETE)
91 {
92 m_status=FIST_DLERROR;
93 m_head.clear();
94 m_head.frontLine=string("HTTP/1.1 ")+sReason;
95 m_head.type=header::ANSWER;
96
97 if (cfg::debug&log::LOG_MORE)
98 log::misc(string("Download of ")+m_sPathRel+" aborted");
99 }
100 Truncate2checkedSize();
101 checkforceclose(m_filefd);
102 }
103
GetTransferCount()104 uint64_t fileitem::GetTransferCount()
105 {
106 setLockGuard;
107 uint64_t ret=m_nIncommingCount;
108 m_nIncommingCount=0;
109 return ret;
110 }
111
GetFileFd()112 int fileitem::GetFileFd() {
113 LOGSTART("fileitem::GetFileFd");
114 setLockGuard;
115
116 ldbg("Opening " << m_sPathRel);
117 int fd=open(SZABSPATH(m_sPathRel), O_RDONLY);
118
119 #ifdef HAVE_FADVISE
120 // optional, experimental
121 if(fd>=0)
122 posix_fadvise(fd, 0, m_nSizeChecked, POSIX_FADV_SEQUENTIAL);
123 #endif
124
125 return fd;
126 }
127
GetFileSize(cmstring & path,off_t defret)128 off_t GetFileSize(cmstring & path, off_t defret)
129 {
130 struct stat stbuf;
131 return (0==::stat(path.c_str(), &stbuf)) ? stbuf.st_size : defret;
132 }
133
134
ResetCacheState()135 void fileitem::ResetCacheState()
136 {
137 setLockGuard;
138 m_nSizeSeen = 0;
139 m_nSizeChecked = 0;
140 m_status = FIST_FRESH;
141 m_bAllowStoreData = true;
142 m_head.clear();
143 }
144
Setup(bool bCheckFreshness)145 fileitem::FiStatus fileitem::Setup(bool bCheckFreshness)
146 {
147 LOGSTART2("fileitem::Setup", bCheckFreshness);
148
149 setLockGuard;
150
151 if(m_status>FIST_FRESH)
152 return m_status;
153
154 m_status=FIST_INITED;
155 m_bCheckFreshness = bCheckFreshness;
156
157 cmstring sPathAbs(CACHE_BASE+m_sPathRel);
158
159 if(m_head.LoadFromFile(sPathAbs+".head") >0 && m_head.type==header::ANSWER )
160 {
161 if(200 != m_head.getStatus())
162 goto error_clean;
163
164
165 LOG("good head");
166
167 m_nSizeSeen=GetFileSize(sPathAbs, 0);
168
169 // some plausibility checks
170 if(m_bCheckFreshness)
171 {
172 const char *p=m_head.h[header::LAST_MODIFIED];
173 if(!p)
174 goto error_clean; // suspicious, cannot use it
175 LOG("check freshness, last modified: " << p );
176
177 // that will cause check by if-mo-only later, needs to be sure about the size here
178 if(cfg::vrangeops == 0
179 && m_nSizeSeen != atoofft(m_head.h[header::CONTENT_LENGTH], -17))
180 {
181 m_nSizeSeen = 0;
182 }
183 }
184 else
185 {
186 // non-volatile files, so could accept the length, do some checks first
187 const char *pContLen=m_head.h[header::CONTENT_LENGTH];
188 if(pContLen)
189 {
190 off_t nContLen=atoofft(pContLen); // if it's 0 then we assume it's 0
191
192 // file larger than it could ever be?
193 if(nContLen < m_nSizeSeen)
194 goto error_clean;
195
196
197 LOG("Content-Length has a sane range");
198
199 m_nSizeChecked=m_nSizeSeen;
200
201 // is it complete? and 0 value also looks weird, try to verify later
202 if(m_nSizeSeen == nContLen && nContLen>0)
203 m_status=FIST_COMPLETE;
204 }
205 else
206 {
207 // no content length known, assume it's ok
208 m_nSizeChecked=m_nSizeSeen;
209 }
210 }
211 }
212 else // -> no .head file
213 {
214 // maybe there is some left-over without head file?
215 // Don't thrust volatile data, but otherwise try to reuse?
216 if(!bCheckFreshness)
217 m_nSizeSeen=GetFileSize(sPathAbs, 0);
218 }
219 LOG("resulting status: " << (int) m_status);
220 return m_status;
221
222 error_clean:
223 ::unlink((sPathAbs+".head").c_str());
224 m_head.clear();
225 m_nSizeSeen=0;
226 m_status=FIST_INITED;
227 return m_status; // unuseable, to be redownloaded
228 }
229
CheckUsableRange_unlocked(off_t nRangeLastByte)230 bool fileitem::CheckUsableRange_unlocked(off_t nRangeLastByte)
231 {
232 if(m_status == FIST_COMPLETE)
233 return true;
234 if(m_status < FIST_INITED || m_status > FIST_COMPLETE)
235 return false;
236 if(m_status >= FIST_DLGOTHEAD)
237 return nRangeLastByte > m_nSizeChecked;
238
239 // special exceptions for static files
240 return (m_status == FIST_INITED && !m_bCheckFreshness
241 && m_nSizeSeen>0 && nRangeLastByte >=0 && nRangeLastByte <m_nSizeSeen
242 && atoofft(m_head.h[header::CONTENT_LENGTH], -255) > nRangeLastByte);
243 }
244
SetupClean(bool bForce)245 bool fileitem::SetupClean(bool bForce)
246 {
247 setLockGuard;
248
249 if(bForce)
250 {
251 if(m_status>FIST_FRESH)
252 {
253 m_status = FIST_DLERROR;
254 m_head.frontLine="HTTP/1.1 500 FIXME, DEAD ITEM";
255 }
256 }
257 else
258 {
259 if(m_status>FIST_FRESH)
260 return false;
261 m_status=FIST_INITED;
262 }
263 cmstring sPathAbs(SABSPATH(m_sPathRel));
264 cmstring sPathHead(sPathAbs+".head");
265 // header allowed to be lost in process...
266 // if(unlink(sPathHead.c_str()))
267 // ::ignore_value(::truncate(sPathHead.c_str(), 0));
268 acng::ignore_value(::truncate(sPathAbs.c_str(), 0));
269 Cstat stf(sPathAbs);
270 if(stf && stf.st_size>0)
271 return false; // didn't work. Permissions? Anyhow, too dangerous to act on this now
272 header h;
273 h.LoadFromFile(sPathHead);
274 h.del(header::CONTENT_LENGTH);
275 h.del(header::CONTENT_TYPE);
276 h.del(header::LAST_MODIFIED);
277 h.del(header::XFORWARDEDFOR);
278 h.del(header::CONTENT_RANGE);
279 h.StoreToFile(sPathHead);
280 // if(0==stat(sPathHead.c_str(), &stf) && stf.st_size >0)
281 // return false; // that's weird too, header still exists with real size
282 m_head.clear();
283 m_nSizeSeen=m_nSizeChecked=0;
284
285 return true;
286 }
287
SetupComplete()288 void fileitem::SetupComplete()
289 {
290 setLockGuard;
291 notifyAll();
292 m_nSizeChecked = m_nSizeSeen;
293 m_status = FIST_COMPLETE;
294 }
295
UpdateHeadTimestamp()296 void fileitem::UpdateHeadTimestamp()
297 {
298 if(m_sPathRel.empty())
299 return;
300 utimes(SZABSPATH(m_sPathRel + ".head"), nullptr);
301 }
302
WaitForFinish(int * httpCode)303 fileitem::FiStatus fileitem::WaitForFinish(int *httpCode)
304 {
305 lockuniq g(this);
306 while(m_status<FIST_COMPLETE)
307 wait(g);
308 if(httpCode)
309 *httpCode=m_head.getStatus();
310 return m_status;
311 }
312
_LogWithErrno(const char * msg,const string & sFile)313 inline void _LogWithErrno(const char *msg, const string & sFile)
314 {
315 tErrnoFmter f;
316 log::err(tSS() << sFile <<
317 " storage error [" << msg << "], last errno: " << f);
318 }
319
320 #ifndef MINIBUILD
321
DownloadStartedStoreHeader(const header & h,size_t hDataLen,const char * pNextData,bool bForcedRestart,bool & bDoCleanRetry)322 bool fileitem_with_storage::DownloadStartedStoreHeader(const header & h, size_t hDataLen,
323 const char *pNextData,
324 bool bForcedRestart, bool &bDoCleanRetry)
325 {
326 LOGSTART("fileitem::DownloadStartedStoreHeader");
327 auto SETERROR = [&](LPCSTR x) {
328 m_bAllowStoreData=false;
329 m_head.frontLine=mstring("HTTP/1.1 ")+x;
330 m_head.set(header::XORIG, h.h[header::XORIG]);
331 m_status=FIST_DLERROR; m_nTimeDlDone=GetTime();
332 _LogWithErrno(x, m_sPathRel);
333 };
334
335 auto withError = [&](LPCSTR x) {
336 SETERROR(x);
337 return false;
338 };
339
340 setLockGuard;
341
342 USRDBG( "Download started, storeHeader for " << m_sPathRel << ", current status: " << (int) m_status);
343
344 if(m_status >= FIST_COMPLETE)
345 {
346 USRDBG( "Download was completed or aborted, not restarting before expiration");
347 return false;
348 }
349
350 // conflict with another thread's download attempt? Deny, except for a forced restart
351 if (m_status > FIST_DLPENDING && !bForcedRestart)
352 return false;
353
354 if(m_bCheckFreshness)
355 m_nTimeDlStarted = GetTime();
356
357 m_nIncommingCount+=hDataLen;
358
359 // optional optimization: hints for the filesystem resp. kernel
360 off_t hint_start(0), hint_length(0);
361
362 // status will change, most likely... ie. return withError action
363 notifyAll();
364
365 cmstring sPathAbs(CACHE_BASE+m_sPathRel);
366 string sHeadPath=sPathAbs + ".head";
367
368 auto withErrorAndKillFile = [&](LPCSTR x)
369 {
370 SETERROR(x);
371 if(m_filefd>=0)
372 {
373 #if _POSIX_SYNCHRONIZED_IO > 0
374 fsync(m_filefd);
375 #endif
376 Truncate2checkedSize();
377 forceclose(m_filefd);
378 }
379
380 LOG("Deleting " << sPathAbs);
381 ::unlink(sPathAbs.c_str());
382 ::unlink(sHeadPath.c_str());
383
384 m_status=FIST_DLERROR;
385 m_nTimeDlDone=GetTime();
386 return false;
387 };
388
389 int serverStatus = h.getStatus();
390 #if 0
391 #warning FIXME
392 static UINT fc=1;
393 if(!(++fc % 4))
394 {
395 serverStatus = 416;
396 }
397 #endif
398 switch(serverStatus)
399 {
400 case 200:
401 {
402 if(m_status < FIST_DLGOTHEAD)
403 bForcedRestart = false; // behave normally, set all data
404
405 if (bForcedRestart)
406 {
407 if (m_nSizeChecked != 0)
408 {
409 /* shouldn't be here, server should have resumed at the previous position.
410 * Most likely the remote file was modified after the download started.
411 */
412 //USRDBG( "state: " << m_status << ", m_nsc: " << m_nSizeChecked);
413 return withError("500 Failed to resume remote download");
414 }
415 if(h.h[header::CONTENT_LENGTH] && atoofft(h.h[header::CONTENT_LENGTH])
416 != atoofft(h.h[header::CONTENT_LENGTH], -2))
417 {
418 return withError("500 Failed to resume remote download, bad length");
419 }
420 m_head.set(header::XORIG, h.h[header::XORIG]);
421 }
422 else
423 {
424 m_nSizeChecked=0;
425 m_head=h;
426 }
427 hint_length=atoofft(h.h[header::CONTENT_LENGTH], 0);
428 break;
429 }
430 case 206:
431 {
432
433 if(m_nSizeSeen<=0 && m_nRangeLimit<0)
434 {
435 // wtf? Cannot have requested partial content
436 return withError("500 Unexpected Partial Response");
437 }
438 /*
439 * Range: bytes=453291-
440 * ...
441 * Content-Length: 7271829
442 * Content-Range: bytes 453291-7725119/7725120
443 */
444 const char *p=h.h[header::CONTENT_RANGE];
445 if(!p)
446 return withError("500 Missing Content-Range in Partial Response");
447 off_t myfrom, myto, mylen;
448 int n=sscanf(p, "bytes " OFF_T_FMT "-" OFF_T_FMT "/" OFF_T_FMT, &myfrom, &myto, &mylen);
449 if(n<=0)
450 n=sscanf(p, "bytes=" OFF_T_FMT "-" OFF_T_FMT "/" OFF_T_FMT, &myfrom, &myto, &mylen);
451
452 ldbg("resuming? n: "<< n << " und myfrom: " <<myfrom <<
453 " und myto: " << myto << " und mylen: " << mylen);
454 if(n!=3 // check for nonsense
455 || (m_nSizeSeen>0 && myfrom != m_nSizeSeen-1)
456 || (m_nRangeLimit>=0 && myto != m_nRangeLimit)
457 || myfrom<0 || mylen<0
458 )
459 {
460 return withError("500 Server reports unexpected range");
461 }
462
463 m_nSizeChecked=myfrom;
464
465 hint_start=myfrom;
466 hint_length=mylen;
467
468 m_head=h;
469 m_head.frontLine="HTTP/1.1 200 OK";
470 m_head.del(header::CONTENT_RANGE);
471 m_head.set(header::CONTENT_LENGTH, mylen);
472 m_head.set(header::XORIG, h.h[header::XORIG]);
473
474 // target opened before? close it, will reopen&seek later
475 if (bForcedRestart)
476 {
477 Truncate2checkedSize();
478 checkforceclose(m_filefd);
479 }
480
481 // special optimization; if "-1 trick" was used then maybe don't reopen that file for writing later
482 if(m_bCheckFreshness && pNextData && m_nSizeSeen == mylen && m_nSizeChecked == mylen-1)
483 {
484 int fd=open(sPathAbs.c_str(), O_RDONLY);
485 if(fd>=0)
486 {
487 if(m_nSizeChecked==lseek(fd, m_nSizeChecked, SEEK_SET))
488 {
489 char c;
490 if(1 == read(fd, &c, 1) && c == *pNextData)
491 {
492 if(cfg::debug & log::LOG_DEBUG)
493 log::err(tSS() << "known data hit, don't write to: "<< m_sPathRel);
494 m_bAllowStoreData=false;
495 m_nSizeChecked=mylen;
496 }
497 }
498 // XXX: optimize that, open as RW if possible and keep the file open for writing
499 forceclose(fd);
500 }
501 }
502 break;
503 }
504 case 416:
505 // that's bad; it cannot have been completed before (the -1 trick)
506 // however, proxy servers with v3r4 cl3v3r caching strategy can cause that
507 // if if-mo-since is used and they don't like it, so attempt a retry in this case
508 if(m_nSizeChecked == 0)
509 {
510 USRDBG( "Peer denied to resume previous download (transient error) " << m_sPathRel );
511 m_nSizeSeen = 0;
512 bDoCleanRetry=true;
513 return false;
514 }
515 else
516 {
517 // -> kill cached file ASAP
518 m_bAllowStoreData=false;
519 m_head.copy(h, header::XORIG);
520 return withErrorAndKillFile("503 Server disagrees on file size, cleaning up");
521 }
522 break;
523 default:
524 m_head.type=header::ANSWER;
525 m_head.copy(h, header::XORIG);
526 m_head.copy(h, header::LOCATION);
527 if(bForcedRestart)
528 {
529 // got an error from the replacement mirror? cannot handle it properly
530 // because some job might already have started returning the data
531 USRDBG( "Cannot restart, HTTP code: " << serverStatus);
532 return withError(h.getCodeMessage());
533 }
534
535 m_bAllowStoreData=false;
536 // have a clean header with just the error message
537 m_head.frontLine=h.frontLine;
538 m_head.set(header::CONTENT_LENGTH, "0");
539 if(m_status>FIST_DLGOTHEAD)
540 {
541 // oh shit. Client may have already started sending it. Prevent such trouble in future.
542 unlink(sHeadPath.c_str());
543 }
544 }
545
546 if(cfg::debug & log::LOG_MORE)
547 log::misc(string("Download of ")+m_sPathRel+" started");
548
549 if(m_bAllowStoreData)
550 {
551 // using adaptive Delete-Or-Replace-Or-CopyOnWrite strategy
552
553 MoveRelease2Sidestore();
554
555 // First opening the file to be sure that it can be written. Header storage is the critical point,
556 // every error after that leads to full cleanup to not risk inconsistent file contents
557
558 int flags = O_WRONLY | O_CREAT | O_BINARY;
559 struct stat stbuf;
560
561 mkbasedir(sPathAbs);
562
563 m_filefd=open(sPathAbs.c_str(), flags, cfg::fileperms);
564 ldbg("file opened?! returned: " << m_filefd);
565
566 // self-recovery from cache poisoned with files with wrong permissions
567 if (m_filefd<0)
568 {
569 if(m_nSizeChecked>0) // OOOH CRAP! CANNOT APPEND HERE! Do what's still possible.
570 {
571 string temp=sPathAbs+".tmp";
572 if(FileCopy(sPathAbs, temp) && 0==unlink(sPathAbs.c_str()) )
573 {
574 if(0!=rename(temp.c_str(), sPathAbs.c_str()))
575 return withError("503 Cannot rename files");
576
577 // be sure about that
578 if(0!=stat(sPathAbs.c_str(), &stbuf) || stbuf.st_size!=m_nSizeSeen)
579 return withError("503 Cannot copy file parts, filesystem full?");
580
581 m_filefd=open(sPathAbs.c_str(), flags, cfg::fileperms);
582 ldbg("file opened after copying around: ");
583 }
584 else
585 return withError((tSS()<<"503 Cannot store or remove files in "
586 << GetDirPart(sPathAbs)).c_str());
587 }
588 else
589 {
590 unlink(sPathAbs.c_str());
591 m_filefd=open(sPathAbs.c_str(), flags, cfg::fileperms);
592 ldbg("file force-opened?! returned: " << m_filefd);
593 }
594 }
595
596 if (m_filefd<0)
597 {
598 tErrnoFmter efmt("503 Cache storage error - ");
599 #ifdef DEBUG
600 return withError((efmt+sPathAbs).c_str());
601 #else
602 return withError(efmt.c_str());
603 #endif
604 }
605
606 if(0!=fstat(m_filefd, &stbuf) || !S_ISREG(stbuf.st_mode))
607 return withErrorAndKillFile("503 Not a regular file");
608
609 // this makes sure not to truncate file while it's mmaped
610 auto tempLock = TFileShrinkGuard::Acquire(stbuf);
611
612 // crop, but only if the new size is smaller. MUST NEVER become larger (would fill with zeros)
613 if(m_nSizeSeen != 0 && m_nSizeChecked <= m_nSizeSeen)
614 {
615 if(0==Truncate2checkedSize())
616 {
617 #if ( (_POSIX_C_SOURCE >= 199309L || _XOPEN_SOURCE >= 500) && _POSIX_SYNCHRONIZED_IO > 0)
618 fdatasync(m_filefd);
619 #endif
620 }
621 else
622 return withErrorAndKillFile("503 Cannot change file size");
623 }
624 else if(m_nSizeChecked>m_nSizeSeen) // should never happen and caught by the checks above
625 return withErrorAndKillFile("503 Internal error on size checking");
626 // else... nothing to fix since the expectation==reality
627
628 // falloc_helper(m_filefd, hint_start, hint_length);
629 if(m_nSizeSeen < (off_t) cfg::allocspace)
630 falloc_helper(m_filefd, 0, min(hint_start+hint_length, (off_t)cfg::allocspace));
631 /*
632 * double-check the docs, this is probably not relevant for writting
633 #ifdef HAVE_FADVISE
634 // optional, experimental
635 posix_fadvise(m_filefd, hint_start, hint_length, POSIX_FADV_SEQUENTIAL);
636 #endif
637 */
638 ldbg("Storing header as "+sHeadPath);
639 int count=m_head.StoreToFile(sHeadPath);
640
641 if(count<0)
642 return withErrorAndKillFile( (-count!=ENOSPC
643 ? "503 Cache storage error" : "503 OUT OF DISK SPACE"));
644
645 // double-check the sane state
646 if(0!=fstat(m_filefd, &stbuf) || stbuf.st_size!=m_nSizeChecked)
647 return withErrorAndKillFile("503 Inconsistent file state");
648
649 if(m_nSizeChecked!=lseek(m_filefd, m_nSizeChecked, SEEK_SET))
650 return withErrorAndKillFile("503 IO error, positioning");
651 }
652
653 m_status=FIST_DLGOTHEAD;
654 return true;
655 }
656
StoreFileData(const char * data,unsigned int size)657 bool fileitem_with_storage::StoreFileData(const char *data, unsigned int size)
658 {
659 setLockGuard;
660
661 LOGSTART2("fileitem::StoreFileData", "status: " << (int) m_status << ", size: " << size);
662
663 // something might care, most likely... also about BOUNCE action
664 notifyAll();
665
666 m_nIncommingCount+=size;
667
668 if(m_status > FIST_COMPLETE || m_status < FIST_DLGOTHEAD)
669 {
670 ldbg("StoreFileData rejected, status: " << (int) m_status)
671 return false;
672 }
673
674 if (size==0)
675 {
676 if(FIST_COMPLETE == m_status)
677 {
678 LOG("already completed");
679 }
680 else
681 {
682 m_status = FIST_COMPLETE;
683 m_nTimeDlDone=GetTime();
684
685 if (cfg::debug & log::LOG_MORE)
686 log::misc(tSS() << "Download of " << m_sPathRel << " finished");
687
688 // we are done! Fix header from chunked transfers?
689 if (m_filefd >= 0 && !m_head.h[header::CONTENT_LENGTH])
690 {
691 m_head.set(header::CONTENT_LENGTH, m_nSizeChecked);
692 m_head.StoreToFile(CACHE_BASE + m_sPathRel + ".head");
693 }
694 }
695 }
696 else
697 {
698 m_status = FIST_DLRECEIVING;
699
700 if (m_bAllowStoreData && m_filefd>=0)
701 {
702 while(size>0)
703 {
704 int r=write(m_filefd, data, size);
705 if(r<0)
706 {
707 if(EINTR==errno || EAGAIN==errno)
708 continue;
709 tErrnoFmter efmt("HTTP/1.1 503 ");
710 m_head.frontLine = efmt;
711 m_status=FIST_DLERROR;
712 // message will be set by the caller
713 m_nTimeDlDone=GetTime();
714 _LogWithErrno(efmt.c_str(), m_sPathRel);
715 return false;
716 }
717 m_nSizeChecked+=r;
718 size-=r;
719 data+=r;
720 }
721 }
722 }
723
724 // needs to remember the good size, just in case the DL is resumed (restarted in hot state)
725 if(m_nSizeSeen < m_nSizeChecked)
726 m_nSizeSeen = m_nSizeChecked;
727
728 return true;
729 }
730
~fileItemMgmt()731 fileItemMgmt::~fileItemMgmt()
732 {
733 LOGSTART("fileItemMgmt::~fileItemMgmt");
734 Unreg();
735 }
736
Unreg()737 inline void fileItemMgmt::Unreg()
738 {
739 LOGSTART("fileItemMgmt::Unreg");
740
741 if(!m_ptr) // unregistered before?
742 return;
743
744 lockguard managementLock(mapItemsMx);
745
746 // invalid or not globally registered?
747 if(m_ptr->m_globRef == mapItems.end())
748 return;
749
750 auto local_ptr(m_ptr); // might disappear
751 lockguard fitemLock(*local_ptr);
752
753 if ( -- m_ptr->usercount <= 0)
754 {
755 if(m_ptr->m_status < fileitem::FIST_COMPLETE && m_ptr->m_status != fileitem::FIST_INITED)
756 {
757 ldbg("usercount dropped to zero while downloading?: " << (int) m_ptr->m_status);
758 }
759
760 // some file items will be held ready for some time
761 time_t when(0);
762 if (MAXTEMPDELAY && m_ptr->m_bCheckFreshness &&
763 m_ptr->m_status == fileitem::FIST_COMPLETE &&
764 ((when=m_ptr->m_nTimeDlStarted+MAXTEMPDELAY) > GetTime()))
765 {
766 g_victor.ScheduleFor(when, cleaner::TYPE_EXFILEITEM);
767 return;
768 }
769
770 // nothing, let's put the item into shutdown state
771 m_ptr->m_status = fileitem::FIST_DLSTOP;
772 m_ptr->m_head.frontLine="HTTP/1.1 500 Cache file item expired";
773 m_ptr->notifyAll();
774
775 LOG("*this is last entry, deleting dl/fi mapping");
776 mapItems.erase(m_ptr->m_globRef);
777 m_ptr->m_globRef = mapItems.end();
778
779 // make sure it's not double-unregistered accidentally!
780 m_ptr.reset();
781 }
782 }
783
784
PrepareRegisteredFileItemWithStorage(cmstring & sPathUnescaped,bool bConsiderAltStore)785 bool fileItemMgmt::PrepareRegisteredFileItemWithStorage(cmstring &sPathUnescaped, bool bConsiderAltStore)
786 {
787 LOGSTART2("fileitem::GetFileItem", sPathUnescaped);
788
789 try
790 {
791 mstring sPathRel(fileitem_with_storage::NormalizePath(sPathUnescaped));
792 lockguard lockGlobalMap(mapItemsMx);
793 tFiGlobMap::iterator it=mapItems.find(sPathRel);
794 if(it!=mapItems.end())
795 {
796 if (bConsiderAltStore)
797 {
798 // detect items that got stuck somehow
799 time_t now(GetTime());
800 time_t extime(now - cfg::stucksecs);
801 if (it->second->m_nTimeDlDone < extime)
802 {
803 // try to find its sibling which is in good state?
804 for (; it!=mapItems.end() && it->first == sPathRel; ++it)
805 {
806 if (it->second->m_nTimeDlDone >= extime)
807 {
808 it->second->usercount++;
809 LOG("Sharing an existing REPLACEMENT file item");
810 m_ptr = it->second;
811 return true;
812 }
813 }
814 // ok, then create a modded name version in the replacement directory
815 mstring sPathRelMod(sPathRel);
816 replaceChars(sPathRelMod, "/\\", '_');
817 sPathRelMod.insert(0, sReplDir + ltos(getpid()) + "_" + ltos(now)+"_");
818 LOG("Registering a new REPLACEMENT file item...");
819 auto sp(make_shared<fileitem_with_storage>(sPathRelMod, 1));
820 sp->m_globRef = mapItems.insert(make_pair(sPathRel, sp));
821 m_ptr = sp;
822 return true;
823 }
824 }
825 LOG("Sharing existing file item");
826 it->second->usercount++;
827 m_ptr = it->second;
828 return true;
829 }
830 LOG("Registering the NEW file item...");
831 auto sp(make_shared<fileitem_with_storage>(sPathRel, 1));
832 sp->m_globRef = mapItems.insert(make_pair(sPathRel, sp));
833 //lockGlobalMap.unLock();
834 m_ptr = sp;
835 return true;
836 }
837 catch(std::bad_alloc&)
838 {
839 }
840 return false;
841 }
842
843 // make the fileitem globally accessible
RegisterFileItem(tFileItemPtr spCustomFileItem)844 bool fileItemMgmt::RegisterFileItem(tFileItemPtr spCustomFileItem)
845 {
846 LOGSTART2("fileitem::RegisterFileItem", spCustomFileItem->m_sPathRel);
847
848 if (!spCustomFileItem || spCustomFileItem->m_sPathRel.empty())
849 return false;
850
851 Unreg();
852
853 lockguard lockGlobalMap(mapItemsMx);
854
855 if(ContHas(mapItems, spCustomFileItem->m_sPathRel))
856 return false; // conflict, another agent is already active
857
858 spCustomFileItem->m_globRef = mapItems.emplace(spCustomFileItem->m_sPathRel,
859 spCustomFileItem);
860 spCustomFileItem->usercount=1;
861 m_ptr = spCustomFileItem;
862 return true;
863 }
864
RegisterFileitemLocalOnly(fileitem * replacement)865 void fileItemMgmt::RegisterFileitemLocalOnly(fileitem* replacement)
866 {
867 LOGSTART2("fileItemMgmt::ReplaceWithLocal", replacement);
868 Unreg();
869 m_ptr.reset(replacement);
870 }
871
872
873 // this method is supposed to be awaken periodically and detects items with ref count manipulated by
874 // the request storm prevention mechanism. Items shall be be dropped after some time if no other
875 // thread but us is using them.
BackgroundCleanup()876 time_t fileItemMgmt::BackgroundCleanup()
877 {
878 LOGSTART2s("fileItemMgmt::BackgroundCleanup", GetTime());
879 lockguard lockGlobalMap(mapItemsMx);
880 tFiGlobMap::iterator it, here;
881
882 time_t now=GetTime();
883 time_t oldestGet = END_OF_TIME;
884 time_t expBefore = now - MAXTEMPDELAY;
885
886 for(it=mapItems.begin(); it!=mapItems.end();)
887 {
888 here=it++;
889
890 // became busy again? Ignore this entry, it's new master will take care of the deletion ASAP
891 if(here->second->usercount >0)
892 continue;
893
894 // find and ignore (but remember) the candidate(s) for the next cycle
895 if (here->second->m_nTimeDlStarted > expBefore)
896 {
897 oldestGet = std::min(time_t(here->second->m_nTimeDlStarted), oldestGet);
898 continue;
899 }
900
901 // ok, unused and delay is over. Destroy with the same sequence as mgmt destructor does,
902 // care about life time.
903 tFileItemPtr local_ptr(here->second);
904 {
905 lockguard g(*local_ptr);
906 local_ptr->m_status = fileitem::FIST_DLSTOP;
907 local_ptr->m_globRef = mapItems.end();
908 mapItems.erase(here);
909 }
910 local_ptr->notifyAll();
911 }
912
913 if(oldestGet == END_OF_TIME)
914 return oldestGet;
915
916 ldbg(oldestGet);
917
918 // preserving a few seconds to catch more of them in the subsequent run
919 return std::max(oldestGet + MAXTEMPDELAY, GetTime()+8);
920 }
921
SendData(int out_fd,int in_fd,off_t & nSendPos,size_t count)922 ssize_t fileitem_with_storage::SendData(int out_fd, int in_fd, off_t &nSendPos, size_t count)
923 {
924 #ifndef HAVE_LINUX_SENDFILE
925 return sendfile_generic(out_fd, in_fd, &nSendPos, count);
926 #else
927 ssize_t r=sendfile(out_fd, in_fd, &nSendPos, count);
928
929 if(r<0 && (errno==ENOSYS || errno==EINVAL))
930 return sendfile_generic(out_fd, in_fd, &nSendPos, count);
931 else
932 return r;
933 #endif
934 }
935
dump_status()936 void fileItemMgmt::dump_status()
937 {
938 tSS fmt;
939 log::err("File descriptor table:\n");
940 for(const auto& item : mapItems)
941 {
942 fmt.clear();
943 fmt << "FREF: " << item.first << " [" << item.second->usercount << "]:\n";
944 if(! item.second)
945 {
946 fmt << "\tBAD REF!\n";
947 continue;
948 }
949 else
950 {
951 fmt << "\t" << item.second->m_sPathRel
952 << "\n\tDlRefCount: " << item.second->m_nDlRefsCount
953 << "\n\tState: " << (int) item.second->m_status
954 << "\n\tFilePos: " << item.second->m_nIncommingCount << " , "
955 << item.second->m_nRangeLimit << " , "
956 << item.second->m_nSizeChecked << " , "
957 << item.second->m_nSizeSeen
958 << "\n\tGotAt: " << item.second->m_nTimeDlStarted << "\n\n";
959 }
960 log::err(fmt.c_str(), nullptr);
961 }
962 log::flush();
963 }
964
~fileitem_with_storage()965 fileitem_with_storage::~fileitem_with_storage()
966 {
967 Truncate2checkedSize();
968
969 if(startsWith(m_sPathRel, sReplDir))
970 {
971 ::unlink(SZABSPATH(m_sPathRel));
972 ::unlink((SABSPATH(m_sPathRel)+".head").c_str());
973 }
974 }
975
Truncate2checkedSize()976 int fileitem_with_storage::Truncate2checkedSize()
977 {
978 if(m_filefd == -1 || m_nSizeChecked<0)
979 return -1;
980 return ftruncate(m_filefd, m_nSizeChecked);
981 }
982
983 // special file? When it's rewritten from start, save the old version instead
MoveRelease2Sidestore()984 int fileitem_with_storage::MoveRelease2Sidestore()
985 {
986 if(m_nSizeChecked)
987 return 0;
988 if(!endsWithSzAr(m_sPathRel, "/InRelease") && !endsWithSzAr(m_sPathRel, "/Release"))
989 return 0;
990 auto tgtDir = CACHE_BASE + cfg::privStoreRelSnapSufix + sPathSep + GetDirPart(m_sPathRel);
991 mkdirhier(tgtDir);
992 auto srcAbs = CACHE_BASE + m_sPathRel;
993 Cstat st(srcAbs);
994 auto sideFileAbs = tgtDir + ltos(st.st_ino) + ltos(st.st_mtim.tv_sec) + ltos(st.st_mtim.tv_nsec);
995 return FileCopy(srcAbs, sideFileAbs);
996 //return rename(srcAbs.c_str(), sideFileAbs.c_str());
997 }
998
999 #endif // MINIBUILD
1000
1001 }
1002