1 /*
2 * OpenClonk, http://www.openclonk.org
3 *
4 * Copyright (c) 2001-2009, RedWolf Design GmbH, http://www.clonk.de/
5 * Copyright (c) 2009-2016, The OpenClonk Team and contributors
6 *
7 * Distributed under the terms of the ISC license; see accompanying file
8 * "COPYING" for details.
9 *
10 * "Clonk" is a registered trademark of Matthes Bender, used with permission.
11 * See accompanying file "TRADEMARK" for details.
12 *
13 * To redistribute this file separately, substitute the full license texts
14 * for the above references.
15 */
16 #include "C4Include.h"
17 #include "network/C4Network2Res.h"
18
19 #include "c4group/C4Components.h"
20 #include "c4group/C4Group.h"
21 #include "control/C4GameControl.h"
22 #include "lib/C4Random.h"
23 #include "game/C4Application.h"
24
25 #include <sys/types.h>
26 #include <sys/stat.h>
27 #ifdef _WIN32
28 #include <direct.h>
29 #endif
30
31 #ifdef _MSC_VER
32 #define snprintf _snprintf
33 #pragma warning (disable : 4355)
34 #endif
35
36 // compile debug options
37 // #define C4NET2RES_LOAD_ALL
38 #define C4NET2RES_DEBUG_LOG
39
40 // helper
41
42 class DirSizeHelper
43 {
44 static uint32_t iSize, iMaxSize;
Helper(const char * szPath)45 static bool Helper(const char *szPath)
46 {
47 if (szPath[SLen(szPath)-1] == '.')
48 return false;
49 if (iSize > iMaxSize)
50 return false;
51 if (DirectoryExists(szPath))
52 ForEachFile(szPath, &Helper);
53 else if (FileExists(szPath))
54 iSize += FileSize(szPath);
55 return true;
56 }
57 public:
GetDirSize(const char * szPath,uint32_t * pSize,uint32_t inMaxSize=~0)58 static bool GetDirSize(const char *szPath, uint32_t *pSize, uint32_t inMaxSize = ~0)
59 {
60 // Security
61 if (!pSize) return false;
62 // Fold it
63 iSize = 0; iMaxSize = inMaxSize;
64 ForEachFile(szPath, &Helper);
65 // Return
66 *pSize = iSize;
67 return true;
68 }
69 };
70 uint32_t DirSizeHelper::iSize, DirSizeHelper::iMaxSize;
71
72 // *** C4Network2ResCore
73
C4Network2ResCore()74 C4Network2ResCore::C4Network2ResCore()
75 : iFileSize(~0u), iFileCRC(~0u), iContentsCRC(~0u),
76 iChunkSize(C4NetResChunkSize)
77 {
78 }
79
Set(C4Network2ResType enType,int32_t iResID,const char * strFileName,uint32_t inContentsCRC)80 void C4Network2ResCore::Set(C4Network2ResType enType, int32_t iResID, const char *strFileName, uint32_t inContentsCRC)
81 {
82 // Initialize base data
83 eType = enType; iID = iResID; iDerID = -1;
84 fLoadable = false;
85 iFileSize = iFileCRC = ~0; iContentsCRC = inContentsCRC;
86 iChunkSize = C4NetResChunkSize;
87 FileName.Copy(strFileName);
88 }
89
SetLoadable(uint32_t iSize,uint32_t iCRC)90 void C4Network2ResCore::SetLoadable(uint32_t iSize, uint32_t iCRC)
91 {
92 fLoadable = true;
93 iFileSize = iSize;
94 iFileCRC = iCRC;
95 }
96
Clear()97 void C4Network2ResCore::Clear()
98 {
99 eType = NRT_Null;
100 iID = iDerID = -1;
101 fLoadable = false;
102 FileName.Clear();
103 iFileSize = iFileCRC = iContentsCRC = ~0;
104 fHasFileSHA = false;
105 }
106
107 // C4PacketBase virtuals
108
CompileFunc(StdCompiler * pComp)109 void C4Network2ResCore::CompileFunc(StdCompiler *pComp)
110 {
111 pComp->Value(mkNamingAdapt(mkEnumAdaptT<uint8_t>(eType, C4Network2ResType_EnumMap), "Type", NRT_Null));
112 pComp->Value(mkNamingAdapt(iID, "ID", -1));
113 pComp->Value(mkNamingAdapt(iDerID, "DerID", -1));
114 pComp->Value(mkNamingAdapt(fLoadable, "Loadable", true));
115 if (fLoadable)
116 {
117 pComp->Value(mkNamingAdapt(iFileSize, "FileSize", 0U));
118 pComp->Value(mkNamingAdapt(iFileCRC, "FileCRC", 0U));
119 pComp->Value(mkNamingAdapt(iChunkSize, "ChunkSize", C4NetResChunkSize));
120 if (!iChunkSize) pComp->excCorrupt("zero chunk size");
121 }
122 pComp->Value(mkNamingAdapt(iContentsCRC, "ContentsCRC", 0U));
123 pComp->Value(mkNamingCountAdapt(fHasFileSHA, "FileSHA"));
124 if (fHasFileSHA)
125 pComp->Value(mkNamingAdapt(mkHexAdapt(FileSHA), "FileSHA"));
126 pComp->Value(mkNamingAdapt(mkNetFilenameAdapt(FileName), "Filename", ""));
127 }
128
129 // *** C4Network2ResLoad
130
C4Network2ResLoad(int32_t inChunk,int32_t inByClient)131 C4Network2ResLoad::C4Network2ResLoad(int32_t inChunk, int32_t inByClient)
132 : iChunk(inChunk), Timestamp(time(nullptr)), iByClient(inByClient), pNext(nullptr)
133 {
134
135 }
136
137 C4Network2ResLoad::~C4Network2ResLoad() = default;
138
CheckTimeout()139 bool C4Network2ResLoad::CheckTimeout()
140 {
141 return difftime(time(nullptr), Timestamp) >= C4NetResLoadTimeout;
142 }
143
144 // *** C4Network2ResChunkData
145
146 C4Network2ResChunkData::C4Network2ResChunkData() = default;
147
C4Network2ResChunkData(const C4Network2ResChunkData & Data2)148 C4Network2ResChunkData::C4Network2ResChunkData(const C4Network2ResChunkData &Data2)
149 : C4PacketBase(Data2),
150 iChunkCnt(Data2.getChunkCnt())
151 {
152 // add ranges
153 Merge(Data2);
154 }
155
~C4Network2ResChunkData()156 C4Network2ResChunkData::~C4Network2ResChunkData()
157 {
158 Clear();
159 }
160
operator =(const C4Network2ResChunkData & Data2)161 C4Network2ResChunkData &C4Network2ResChunkData::operator =(const C4Network2ResChunkData &Data2)
162 {
163 // clear, merge
164 SetIncomplete(Data2.getChunkCnt());
165 Merge(Data2);
166 return *this;
167 }
168
SetIncomplete(int32_t inChunkCnt)169 void C4Network2ResChunkData::SetIncomplete(int32_t inChunkCnt)
170 {
171 Clear();
172 // just set total chunk count
173 iChunkCnt = inChunkCnt;
174 }
175
SetComplete(int32_t inChunkCnt)176 void C4Network2ResChunkData::SetComplete(int32_t inChunkCnt)
177 {
178 Clear();
179 // set total chunk count
180 iPresentChunkCnt = iChunkCnt = inChunkCnt;
181 // create one range
182 ChunkRange *pRange = new ChunkRange;
183 pRange->Start = 0; pRange->Length = iChunkCnt;
184 pRange->Next = nullptr;
185 pChunkRanges = pRange;
186 }
187
AddChunk(int32_t iChunk)188 void C4Network2ResChunkData::AddChunk(int32_t iChunk)
189 {
190 AddChunkRange(iChunk, 1);
191 }
192
AddChunkRange(int32_t iStart,int32_t iLength)193 void C4Network2ResChunkData::AddChunkRange(int32_t iStart, int32_t iLength)
194 {
195 // security
196 if (iStart < 0 || iStart + iLength > iChunkCnt || iLength <= 0) return;
197 // find position
198 ChunkRange *pRange, *pPrev;
199 for (pRange = pChunkRanges, pPrev = nullptr; pRange; pPrev = pRange, pRange = pRange->Next)
200 if (pRange->Start >= iStart)
201 break;
202 // create new
203 ChunkRange *pNew = new ChunkRange;
204 pNew->Start = iStart; pNew->Length = iLength;
205 // add to list
206 pNew->Next = pRange;
207 (pPrev ? pPrev->Next : pChunkRanges) = pNew;
208 // counts
209 iPresentChunkCnt += iLength; iChunkRangeCnt++;
210 // check merges
211 if (pPrev && MergeRanges(pPrev))
212 while (MergeRanges(pPrev)) {}
213 else
214 while (MergeRanges(pNew)) {}
215 }
216
Merge(const C4Network2ResChunkData & Data2)217 void C4Network2ResChunkData::Merge(const C4Network2ResChunkData &Data2)
218 {
219 // must have same basis chunk count
220 assert(iChunkCnt == Data2.getChunkCnt());
221 // add ranges
222 for (ChunkRange *pRange = Data2.pChunkRanges; pRange; pRange = pRange->Next)
223 AddChunkRange(pRange->Start, pRange->Length);
224 }
225
Clear()226 void C4Network2ResChunkData::Clear()
227 {
228 iChunkCnt = iPresentChunkCnt = iChunkRangeCnt = 0;
229 // remove all ranges
230 while (pChunkRanges)
231 {
232 ChunkRange *pDelete = pChunkRanges;
233 pChunkRanges = pDelete->Next;
234 delete pDelete;
235 }
236 }
237
GetChunkToRetrieve(const C4Network2ResChunkData & Available,int32_t iLoadingCnt,int32_t * pLoading) const238 int32_t C4Network2ResChunkData::GetChunkToRetrieve(const C4Network2ResChunkData &Available, int32_t iLoadingCnt, int32_t *pLoading) const
239 {
240 // (this version is highly calculation-intensitive, yet the most satisfactory
241 // solution I could find)
242
243 // find everything that should not be retrieved
244 C4Network2ResChunkData ChData; Available.GetNegative(ChData);
245 ChData.Merge(*this);
246 for (int32_t i = 0; i < iLoadingCnt; i++)
247 ChData.AddChunk(pLoading[i]);
248 // nothing to retrieve?
249 if (ChData.isComplete()) return -1;
250 // invert to get everything that should be retrieved
251 C4Network2ResChunkData ChData2; ChData.GetNegative(ChData2);
252 // select chunk (random)
253 int32_t iRetrieveChunk = UnsyncedRandom(ChData2.getPresentChunkCnt());
254 // return
255 return ChData2.getPresentChunk(iRetrieveChunk);
256 }
257
MergeRanges(ChunkRange * pRange)258 bool C4Network2ResChunkData::MergeRanges(ChunkRange *pRange)
259 {
260 // no next entry?
261 if (!pRange || !pRange->Next) return false;
262 // do merge?
263 ChunkRange *pNext = pRange->Next;
264 if (pRange->Start + pRange->Length < pNext->Start) return false;
265 // get overlap
266 int32_t iOverlap = std::min((pRange->Start + pRange->Length) - pNext->Start, pNext->Length);
267 // set new chunk range
268 pRange->Length += pNext->Length - iOverlap;
269 // remove range
270 pRange->Next = pNext->Next;
271 delete pNext;
272 // counts
273 iChunkRangeCnt--; iPresentChunkCnt -= iOverlap;
274 // ok
275 return true;
276 }
277
GetNegative(C4Network2ResChunkData & Target) const278 void C4Network2ResChunkData::GetNegative(C4Network2ResChunkData &Target) const
279 {
280 // clear target
281 Target.SetIncomplete(iChunkCnt);
282 // add all ranges that are missing
283 int32_t iFreeStart = 0;
284 for (ChunkRange *pRange = pChunkRanges; pRange; pRange = pRange->Next)
285 {
286 // add range
287 Target.AddChunkRange(iFreeStart, pRange->Start - iFreeStart);
288 // safe new start
289 iFreeStart = pRange->Start + pRange->Length;
290 }
291 // add last range
292 Target.AddChunkRange(iFreeStart, iChunkCnt - iFreeStart);
293 }
294
getPresentChunk(int32_t iNr) const295 int32_t C4Network2ResChunkData::getPresentChunk(int32_t iNr) const
296 {
297 for (ChunkRange *pRange = pChunkRanges; pRange; pRange = pRange->Next)
298 if (iNr < pRange->Length)
299 return iNr + pRange->Start;
300 else
301 iNr -= pRange->Length;
302 return -1;
303 }
304
CompileFunc(StdCompiler * pComp)305 void C4Network2ResChunkData::CompileFunc(StdCompiler *pComp)
306 {
307 bool deserializing = pComp->isDeserializer();
308 if (deserializing) Clear();
309 // Data
310 pComp->Value(mkNamingAdapt(mkIntPackAdapt(iChunkCnt), "ChunkCnt", 0));
311 pComp->Value(mkNamingAdapt(mkIntPackAdapt(iChunkRangeCnt), "ChunkRangeCnt", 0));
312 // Ranges
313 if (!pComp->Name("Ranges"))
314 pComp->excCorrupt("ResChunk ranges expected!");
315 ChunkRange *pRange = nullptr;
316 for (int32_t i = 0; i < iChunkRangeCnt; i++)
317 {
318 // Create new range / go to next range
319 if (deserializing)
320 pRange = (pRange ? pRange->Next : pChunkRanges) = new ChunkRange;
321 else
322 pRange = pRange ? pRange->Next : pChunkRanges;
323 // Separate
324 if (i) pComp->Separator();
325 // Compile range
326 pComp->Value(mkIntPackAdapt(pRange->Start));
327 pComp->Separator(StdCompiler::SEP_PART2);
328 pComp->Value(mkIntPackAdapt(pRange->Length));
329 }
330 // Terminate list
331 if (deserializing)
332 (pRange ? pRange->Next : pChunkRanges) = nullptr;
333 pComp->NameEnd();
334 }
335
336 // *** C4Network2Res
337
C4Network2Res(C4Network2ResList * pnParent)338 C4Network2Res::C4Network2Res(C4Network2ResList *pnParent)
339 : fDirty(false),
340 fTempFile(false), fStandaloneFailed(false),
341 iRefCnt(0), fRemoved(false),
342 iLastReqTime(0),
343 fLoading(false),
344 pCChunks(nullptr), iDiscoverStartTime(0), pLoads(nullptr), iLoadCnt(0),
345 pNext(nullptr),
346 pParent(pnParent)
347 {
348 szFile[0] = szStandalone[0] = '\0';
349 }
350
~C4Network2Res()351 C4Network2Res::~C4Network2Res()
352 {
353 assert(!pNext);
354 Clear();
355 }
356
SetByFile(const char * strFilePath,bool fTemp,C4Network2ResType eType,int32_t iResID,const char * szResName,bool fSilent)357 bool C4Network2Res::SetByFile(const char *strFilePath, bool fTemp, C4Network2ResType eType, int32_t iResID, const char *szResName, bool fSilent)
358 {
359 CStdLock FileLock(&FileCSec);
360 // default resource name: relative path
361 if (!szResName) szResName = Config.AtRelativePath(strFilePath);
362 SCopy(strFilePath, szFile, sizeof(szFile)-1);
363 // group?
364 C4Group Grp;
365 if (Reloc.Open(Grp, strFilePath))
366 return SetByGroup(&Grp, fTemp, eType, iResID, szResName, fSilent);
367 // so it needs to be a file
368 StdStrBuf szFullFile;
369 if (!Reloc.LocateItem(szFile, szFullFile))
370 { if (!fSilent) LogF("SetByFile: file %s not found!", strFilePath); return false; }
371 // calc checksum
372 uint32_t iCRC32;
373 if (!GetFileCRC(szFullFile.getData(), &iCRC32)) return false;
374 #ifdef C4NET2RES_DEBUG_LOG
375 // log
376 LogSilentF("Network: Resource: complete %d:%s is file %s (%s)", iResID, szResName, szFile, fTemp ? "temp" : "static");
377 #endif
378 // set core
379 Core.Set(eType, iResID, Config.AtRelativePath(szFullFile.getData()), iCRC32);
380 // set own data
381 fDirty = true;
382 fTempFile = fTemp;
383 fStandaloneFailed = false;
384 fRemoved = false;
385 iLastReqTime = time(nullptr);
386 fLoading = false;
387 // ok
388 return true;
389 }
390
SetByGroup(C4Group * pGrp,bool fTemp,C4Network2ResType eType,int32_t iResID,const char * szResName,bool fSilent)391 bool C4Network2Res::SetByGroup(C4Group *pGrp, bool fTemp, C4Network2ResType eType, int32_t iResID, const char *szResName, bool fSilent) // by main thread
392 {
393 Clear();
394 CStdLock FileLock(&FileCSec);
395 // default resource name: relative path
396 StdStrBuf sResName;
397 if (szResName)
398 sResName = szResName;
399 else
400 {
401 StdStrBuf sFullName = pGrp->GetFullName();
402 sResName.Copy(Config.AtRelativePath(sFullName.getData()));
403 }
404 SCopy(pGrp->GetFullName().getData(), szFile, sizeof(szFile)-1);
405 // set core
406 Core.Set(eType, iResID, sResName.getData(), pGrp->EntryCRC32());
407 #ifdef C4NET2RES_DEBUG_LOG
408 // log
409 LogSilentF("Network: Resource: complete %d:%s is file %s (%s)", iResID, sResName.getData(), szFile, fTemp ? "temp" : "static");
410 #endif
411 // set data
412 fDirty = true;
413 fTempFile = fTemp;
414 fStandaloneFailed = false;
415 fRemoved = false;
416 iLastReqTime = time(nullptr);
417 fLoading = false;
418 // ok
419 return true;
420 }
421
SetByCore(const C4Network2ResCore & nCore,bool fSilent,const char * szAsFilename,int32_t iRecursion)422 bool C4Network2Res::SetByCore(const C4Network2ResCore &nCore, bool fSilent, const char *szAsFilename, int32_t iRecursion) // by main thread
423 {
424 StdStrBuf sFilename;
425 // try open local file
426 const char *szFilename = szAsFilename ? szAsFilename : GetC4Filename(nCore.getFileName());
427 if (SetByFile(szFilename, false, nCore.getType(), nCore.getID(), nCore.getFileName(), fSilent))
428 {
429 // check contents checksum
430 if (Core.getContentsCRC() == nCore.getContentsCRC())
431 {
432 // set core
433 fDirty = true;
434 Core = nCore;
435 // ok then
436 return true;
437 }
438 }
439 // get and search for filename without specified folder (e.g., Castle.ocs when the opened game is Easy.ocf\Castle.ocs)
440 const char *szFilenameOnly = GetFilename(szFilename);
441 const char *szFilenameC4 = GetC4Filename(szFilename);
442 if (szFilenameOnly != szFilenameC4)
443 {
444 sFilename.Copy(szFilename, SLen(szFilename) - SLen(szFilenameC4));
445 sFilename.Append(szFilenameOnly);
446 if (SetByCore(nCore, fSilent, szFilenameOnly, Config.Network.MaxResSearchRecursion)) return true;
447 }
448 // if it could still not be set, try within all folders of root (ignoring special folders), and try as file outside the folder
449 // but do not recurse any deeper than set by config (default: One folder)
450 if (iRecursion >= Config.Network.MaxResSearchRecursion) return false;
451 StdStrBuf sSearchPath; const char *szSearchPath;
452 if (!iRecursion)
453 szSearchPath = Config.General.ExePath.getData();
454 else
455 {
456 sSearchPath.Copy(szFilename, SLen(szFilename) - SLen(szFilenameC4));
457 szSearchPath = sSearchPath.getData();
458 }
459 StdStrBuf sNetPath; sNetPath.Copy(Config.Network.WorkPath);
460 char *szNetPath = sNetPath.GrabPointer();
461 TruncateBackslash(szNetPath);
462 sNetPath.Take(szNetPath);
463 for (DirectoryIterator i(szSearchPath); *i; ++i)
464 if (DirectoryExists(*i))
465 if (!*GetExtension(*i)) // directories without extension only
466 if (!szNetPath || !*szNetPath || !ItemIdentical(*i, szNetPath)) // ignore network path
467 {
468 // search for complete name at subpath (e.g. MyFolder\Easy.ocf\Castle.ocs)
469 sFilename.Format("%s%c%s", *i, DirectorySeparator, szFilenameC4);
470 if (SetByCore(nCore, fSilent, sFilename.getData(), iRecursion + 1))
471 return true;
472 }
473 // file could not be found locally
474 return false;
475 }
476
SetLoad(const C4Network2ResCore & nCore)477 bool C4Network2Res::SetLoad(const C4Network2ResCore &nCore) // by main thread
478 {
479 Clear();
480 CStdLock FileLock(&FileCSec);
481 // must be loadable
482 if (!nCore.isLoadable()) return false;
483 // save core, set chunks
484 Core = nCore;
485 Chunks.SetIncomplete(Core.getChunkCnt());
486 // create temporary file
487 if (!pParent->FindTempResFileName(Core.getFileName(), szFile))
488 return false;
489 #ifdef C4NET2RES_DEBUG_LOG
490 // log
491 Application.InteractiveThread.ThreadLogS("Network: Resource: loading %d:%s to file %s", Core.getID(), Core.getFileName(), szFile);
492 #endif
493 // set standalone (result is going to be binary-compatible)
494 SCopy(szFile, szStandalone, sizeof(szStandalone) - 1);
495 // set flags
496 fDirty = false;
497 fTempFile = true;
498 fStandaloneFailed = false;
499 fRemoved = false;
500 iLastReqTime = time(nullptr);
501 fLoading = true;
502 // No discovery yet
503 iDiscoverStartTime = 0;
504 return true;
505 }
506
SetDerived(const char * strName,const char * strFilePath,bool fTemp,C4Network2ResType eType,int32_t iDResID)507 bool C4Network2Res::SetDerived(const char *strName, const char *strFilePath, bool fTemp, C4Network2ResType eType, int32_t iDResID)
508 {
509 Clear();
510 CStdLock FileLock(&FileCSec);
511 // set core
512 Core.Set(eType, C4NetResIDAnonymous, strName, ~0);
513 Core.SetDerived(iDResID);
514 // save file path
515 SCopy(strFilePath, szFile, _MAX_PATH);
516 *szStandalone = '\0';
517 // set flags
518 fDirty = false;
519 fTempFile = fTemp;
520 fStandaloneFailed = false;
521 fRemoved = false;
522 iLastReqTime = time(nullptr);
523 fLoading = false;
524 // Do not set any chunk data - anonymous resources are very likely to change.
525 // Wait for FinishDerived()-call.
526 return true;
527 }
528
ChangeID(int32_t inID)529 void C4Network2Res::ChangeID(int32_t inID)
530 {
531 Core.SetID(inID);
532 }
533
IsBinaryCompatible()534 bool C4Network2Res::IsBinaryCompatible()
535 {
536 // returns wether the standalone of this resource is binary compatible
537 // to the official version (means: matches the file checksum)
538
539 CStdLock FileLock(&FileCSec);
540 // standalone set? ok then (see GetStandalone)
541 if (szStandalone[0]) return true;
542 // is a directory?
543 if (DirectoryExists(szFile))
544 // forget it - if the file is packed now, the creation time and author
545 // won't match.
546 return false;
547 // try to create the standalone
548 return GetStandalone(nullptr, 0, false, false, true);
549 }
550
GetStandalone(char * pTo,int32_t iMaxL,bool fSetOfficial,bool fAllowUnloadable,bool fSilent)551 bool C4Network2Res::GetStandalone(char *pTo, int32_t iMaxL, bool fSetOfficial, bool fAllowUnloadable, bool fSilent)
552 {
553 // already set?
554 if (szStandalone[0])
555 {
556 if (pTo) SCopy(szStandalone, pTo, iMaxL);
557 return true;
558 }
559 // already tried and failed? No point in retrying
560 if (fStandaloneFailed) return false;
561 // not loadable? Wo won't be able to check the standalone as the core will lack the needed information.
562 // the standalone won't be interesting in this case, anyway.
563 if (!fSetOfficial && !Core.isLoadable()) return false;
564 // set flag, so failure below will let future calls fail
565 fStandaloneFailed = true;
566 // lock file
567 CStdLock FileLock(&FileCSec);
568
569 // directory?
570 SCopy(szFile, szStandalone, sizeof(szStandalone)-1);
571 if (DirectoryExists(szFile))
572 {
573 // size check for the directory, if allowed
574 if (fAllowUnloadable)
575 {
576 uint32_t iDirSize;
577 if (!DirSizeHelper::GetDirSize(szFile, &iDirSize, Config.Network.MaxLoadFileSize))
578 { if (!fSilent) LogF("Network: could not get directory size of %s!", szFile); szStandalone[0] = '\0'; return false; }
579 if (iDirSize > uint32_t(Config.Network.MaxLoadFileSize))
580 { if (!fSilent) LogSilentF("Network: %s over size limit, will be marked unloadable!", szFile); szStandalone[0] = '\0'; return false; }
581 }
582 // log - this may take a few seconds
583 if (!fSilent) LogF(LoadResStr("IDS_PRC_NETPACKING"), GetFilename(szFile));
584 // pack inplace?
585 if (!fTempFile)
586 {
587 if (!pParent->FindTempResFileName(szFile, szStandalone))
588 { if (!fSilent) Log("GetStandalone: could not find free name for temporary file!"); szStandalone[0] = '\0'; return false; }
589 if (!C4Group_PackDirectoryTo(szFile, szStandalone))
590 { if (!fSilent) Log("GetStandalone: could not pack directory!"); szStandalone[0] = '\0'; return false; }
591 }
592 else if (!C4Group_PackDirectory(szStandalone))
593 { if (!fSilent) Log("GetStandalone: could not pack directory!"); if (!SEqual(szFile, szStandalone)) EraseDirectory(szStandalone); szStandalone[0] = '\0'; return false; }
594 // make sure directory is packed
595 if (DirectoryExists(szStandalone))
596 { if (!fSilent) Log("GetStandalone: directory hasn't been packed!"); if (!SEqual(szFile, szStandalone)) EraseDirectory(szStandalone); szStandalone[0] = '\0'; return false; }
597 // fallthru
598 }
599
600 // doesn't exist physically?
601 if (!FileExists(szStandalone))
602 {
603 // try C4Group (might be packed)
604 if (!pParent->FindTempResFileName(szFile, szStandalone))
605 { if (!fSilent) Log("GetStandalone: could not find free name for temporary file!"); szStandalone[0] = '\0'; return false; }
606 if (!C4Group_CopyItem(szFile, szStandalone))
607 { if (!fSilent) Log("GetStandalone: could not copy to temporary file!"); szStandalone[0] = '\0'; return false; }
608 }
609
610 // remains missing? give up.
611 if (!FileExists(szStandalone))
612 { if (!fSilent) Log("GetStandalone: file not found!"); szStandalone[0] = '\0'; return false; }
613
614 // do optimizations (delete unneeded entries)
615 if (!OptimizeStandalone(fSilent))
616 { if (!SEqual(szFile, szStandalone)) EraseItem(szStandalone); szStandalone[0] = '\0'; return false; }
617
618 // get file size
619 size_t iSize = FileSize(szStandalone);
620 // size limit
621 if (fAllowUnloadable)
622 if (iSize > uint32_t(Config.Network.MaxLoadFileSize))
623 { if (!fSilent) LogSilentF("Network: %s over size limit, will be marked unloadable!", szFile); szStandalone[0] = '\0'; return false; }
624 // check
625 if (!fSetOfficial && iSize != Core.getFileSize())
626 {
627 // remove file
628 if (!SEqual(szFile, szStandalone)) EraseItem(szStandalone); szStandalone[0] = '\0';
629 // sorry, this version isn't good enough :(
630 return false;
631 }
632
633 // calc checksum
634 uint32_t iCRC32;
635 if (!GetFileCRC(szStandalone, &iCRC32))
636 { if (!fSilent) Log("GetStandalone: could not calculate checksum!"); return false; }
637 // set / check
638 if (!fSetOfficial && iCRC32 != Core.getFileCRC())
639 {
640 // remove file, return
641 if (!SEqual(szFile, szStandalone)) EraseItem(szStandalone); szStandalone[0] = '\0';
642 return false;
643 }
644
645 // we didn't fail
646 fStandaloneFailed = false;
647 // mark resource as loadable and safe file information
648 Core.SetLoadable(iSize, iCRC32);
649 // set up chunk data
650 Chunks.SetComplete(Core.getChunkCnt());
651 // ok
652 return true;
653 }
654
CalculateSHA()655 bool C4Network2Res::CalculateSHA()
656 {
657 // already present?
658 if (Core.hasFileSHA()) return true;
659 // get the file
660 char szStandalone[_MAX_PATH + 1];
661 if (!GetStandalone(szStandalone, _MAX_PATH, false))
662 SCopy(szFile, szStandalone, _MAX_PATH);
663 // get the hash
664 BYTE hash[SHA_DIGEST_LENGTH];
665 if (!GetFileSHA1(szStandalone, hash))
666 return false;
667 // save it back
668 Core.SetFileSHA(hash);
669 // okay
670 return true;
671 }
672
673
Derive()674 C4Network2Res::Ref C4Network2Res::Derive()
675 {
676 // Called before the file is changed. Rescues all files and creates a
677 // new resource for the file. This resource is flagged as "anonymous", as it
678 // has no official core (no res ID, to be exact).
679 // The resource gets its final core when FinishDerive() is called.
680
681 // For security: This doesn't make much sense if the resource is currently being
682 // loaded. So better assume the caller doesn't know what he's doing and check.
683 if (isLoading()) return nullptr;
684
685 CStdLock FileLock(&FileCSec);
686 // Save back original file name
687 char szOrgFile[_MAX_PATH+1];
688 SCopy(szFile, szOrgFile, _MAX_PATH);
689 bool fOrgTempFile = fTempFile;
690
691 // Create a copy of the file, if neccessary
692 if (!*szStandalone || SEqual(szStandalone, szFile))
693 {
694 if (!pParent->FindTempResFileName(szOrgFile, szFile))
695 { Log("Derive: could not find free name for temporary file!"); return nullptr; }
696 if (!C4Group_CopyItem(szOrgFile, szFile))
697 { Log("Derive: could not copy to temporary file!"); return nullptr; }
698 // set standalone
699 if (*szStandalone)
700 SCopy(szFile, szStandalone, _MAX_PATH);
701 fTempFile = true;
702 }
703 else
704 {
705 // Standlone exists: Just set szFile to point on the standlone. It's
706 // assumed that the original file isn't of intrest after this point anyway.
707 SCopy(szStandalone, szFile, _MAX_PATH);
708 fTempFile = true;
709 }
710
711 Application.InteractiveThread.ThreadLogS("Network: Resource: deriving from %d:%s, original at %s", getResID(), Core.getFileName(), szFile);
712
713 // (note: should remove temp file if something fails after this point)
714
715 // create new resource
716 C4Network2Res::Ref pDRes = new C4Network2Res(pParent);
717 if (!pDRes) return nullptr;
718
719 // initialize
720 if (!pDRes->SetDerived(Core.getFileName(), szOrgFile, fOrgTempFile, getType(), getResID()))
721 return nullptr;
722
723 // add to list
724 pParent->Add(pDRes);
725
726 // return new resource
727 return pDRes;
728 }
729
FinishDerive()730 bool C4Network2Res::FinishDerive() // by main thread
731 {
732 // All changes have been made. Register this resource with a new ID.
733
734 // security
735 if (!isAnonymous()) return false;
736
737 CStdLock FileLock(&FileCSec);
738 // Save back data
739 int32_t iDerID = Core.getDerID();
740 char szName[_MAX_PATH+1]; SCopy(Core.getFileName(), szName, _MAX_PATH);
741 char szFileC[_MAX_PATH+1]; SCopy(szFile, szFileC, _MAX_PATH);
742 // Set by file
743 if (!SetByFile(szFileC, fTempFile, getType(), pParent->nextResID(), szName))
744 return false;
745 // create standalone
746 if (!GetStandalone(nullptr, 0, true))
747 return false;
748 // Set ID
749 Core.SetDerived(iDerID);
750 // announce derive
751 pParent->getIOClass()->BroadcastMsg(MkC4NetIOPacket(PID_NetResDerive, Core));
752 // derivation is dirty bussines
753 fDirty = true;
754 // ok
755 return true;
756 }
757
FinishDerive(const C4Network2ResCore & nCore)758 bool C4Network2Res::FinishDerive(const C4Network2ResCore &nCore)
759 {
760 // security
761 if (!isAnonymous()) return false;
762 // Set core
763 Core = nCore;
764 // Set chunks (assume the resource is complete)
765 Chunks.SetComplete(Core.getChunkCnt());
766
767 // Note that the Contents-CRC is /not/ checked. Derivation needs to be
768 // synchronized outside of C4Network2Res.
769
770 // But note that the resource /might/ be binary compatible (though very
771 // unlikely), so do not set fNotBinaryCompatible.
772
773 // ok
774 return true;
775 }
776
OpenAsGrp() const777 C4Group *C4Network2Res::OpenAsGrp() const
778 {
779 C4Group *pnGrp = new C4Group();
780 if (!pnGrp->Open(szFile))
781 {
782 delete pnGrp;
783 return nullptr;
784 }
785 return pnGrp;
786 }
787
Remove()788 void C4Network2Res::Remove()
789 {
790 // schedule for removal
791 fRemoved = true;
792 }
793
SendStatus(C4Network2IOConnection * pTo)794 bool C4Network2Res::SendStatus(C4Network2IOConnection *pTo)
795 {
796 // pack status
797 C4NetIOPacket Pkt = MkC4NetIOPacket(PID_NetResStat, C4PacketResStatus(Core.getID(), Chunks));
798 // to one client?
799 if (pTo)
800 return pTo->Send(Pkt);
801 else
802 {
803 // reset dirty flag
804 fDirty = false;
805 // broadcast status
806 assert(pParent && pParent->getIOClass());
807 return pParent->getIOClass()->BroadcastMsg(Pkt);
808 }
809 }
810
SendChunk(uint32_t iChunk,int32_t iToClient)811 bool C4Network2Res::SendChunk(uint32_t iChunk, int32_t iToClient)
812 {
813 assert(pParent && pParent->getIOClass());
814 if (!szStandalone[0] || iChunk >= Core.getChunkCnt()) return false;
815 // find connection for given client (one of the rare uses of the data connection)
816 C4Network2IOConnection *pConn = pParent->getIOClass()->GetDataConnection(iToClient);
817 if (!pConn) return false;
818 // save last request time
819 iLastReqTime = time(nullptr);
820 // create packet
821 CStdLock FileLock(&FileCSec);
822 C4Network2ResChunk ResChunk;
823 ResChunk.Set(this, iChunk);
824 // send
825 bool fSuccess = pConn->Send(MkC4NetIOPacket(PID_NetResData, ResChunk));
826 pConn->DelRef();
827 return fSuccess;
828 }
829
AddRef()830 void C4Network2Res::AddRef()
831 {
832 ++iRefCnt;
833 }
834
DelRef()835 void C4Network2Res::DelRef()
836 {
837 if (--iRefCnt == 0)
838 delete this;
839 }
840
OnDiscover(C4Network2IOConnection * pBy)841 void C4Network2Res::OnDiscover(C4Network2IOConnection *pBy)
842 {
843 if (!IsBinaryCompatible()) return;
844 // discovered
845 iLastReqTime = time(nullptr);
846 // send status back
847 SendStatus(pBy);
848 }
849
OnStatus(const C4Network2ResChunkData & rChunkData,C4Network2IOConnection * pBy)850 void C4Network2Res::OnStatus(const C4Network2ResChunkData &rChunkData, C4Network2IOConnection *pBy)
851 {
852 if (!fLoading) return;
853 // discovered a source: reset timeout
854 iDiscoverStartTime = 0;
855 // check if the chunk data is valid
856 if (rChunkData.getChunkCnt() != Chunks.getChunkCnt())
857 return;
858 // add chunk data
859 ClientChunks *pChunks;
860 for (pChunks = pCChunks; pChunks; pChunks = pChunks->Next)
861 if (pChunks->ClientID == pBy->getClientID())
862 break;
863 // not found? add
864 if (!pChunks)
865 {
866 pChunks = new ClientChunks();
867 pChunks->Next = pCChunks;
868 pCChunks = pChunks;
869 }
870 pChunks->ClientID = pBy->getClientID();
871 pChunks->Chunks = rChunkData;
872 // check load
873 if (!StartLoad(pChunks->ClientID, pChunks->Chunks))
874 RemoveCChunks(pCChunks);
875 }
876
OnChunk(const C4Network2ResChunk & rChunk)877 void C4Network2Res::OnChunk(const C4Network2ResChunk &rChunk)
878 {
879 if (!fLoading) return;
880 // correct resource?
881 if (rChunk.getResID() != getResID()) return;
882 // add resource data
883 CStdLock FileLock(&FileCSec);
884 bool fSuccess = rChunk.AddTo(this, pParent->getIOClass());
885 #ifdef C4NET2RES_DEBUG_LOG
886 // log
887 Application.InteractiveThread.ThreadLogS("Network: Res: %s chunk %d to resource %s (%s)%s", fSuccess ? "added" : "could not add", rChunk.getChunkNr(), Core.getFileName(), szFile, fSuccess ? "" : "!");
888 #endif
889 if (fSuccess)
890 {
891 // status changed
892 fDirty = true;
893 // remove load waits
894 for (C4Network2ResLoad *pLoad = pLoads, *pNext; pLoad; pLoad = pNext)
895 {
896 pNext = pLoad->Next();
897 if (static_cast<uint32_t>(pLoad->getChunk()) == rChunk.getChunkNr())
898 RemoveLoad(pLoad);
899 }
900 }
901 // complete?
902 if (Chunks.isComplete())
903 EndLoad();
904 // check: start new loads?
905 else
906 StartNewLoads();
907 }
908
DoLoad()909 bool C4Network2Res::DoLoad()
910 {
911 if (!fLoading) return true;
912 // any loads currently active?
913 if (iLoadCnt)
914 {
915 // check for load timeouts
916 int32_t iLoadsRemoved = 0;
917 for (C4Network2ResLoad *pLoad = pLoads, *pNext; pLoad; pLoad = pNext)
918 {
919 pNext = pLoad->Next();
920 if (pLoad->CheckTimeout())
921 {
922 RemoveLoad(pLoad);
923 iLoadsRemoved++;
924 }
925 }
926 // start new loads
927 if (iLoadsRemoved) StartNewLoads();
928 }
929 else
930 {
931 // discover timeout?
932 if (iDiscoverStartTime)
933 if (difftime(time(nullptr), iDiscoverStartTime) > C4NetResDiscoverTimeout)
934 return false;
935 }
936 // ok
937 return true;
938 }
939
NeedsDiscover()940 bool C4Network2Res::NeedsDiscover()
941 {
942 // loading, but no active load sources?
943 if (fLoading && !iLoadCnt)
944 {
945 // set timeout, if this is the first discover
946 if (!iDiscoverStartTime)
947 iDiscoverStartTime = time(nullptr);
948 // do discover
949 return true;
950 }
951 return false;
952 }
953
Clear()954 void C4Network2Res::Clear()
955 {
956 CStdLock FileLock(&FileCSec);
957 // delete files
958 if (fTempFile)
959 if (FileExists(szFile))
960 if (!EraseFile(szFile))
961 LogSilentF("Network: Could not delete temporary resource file (%s)", strerror(errno));
962 if (szStandalone[0] && !SEqual(szFile, szStandalone))
963 if (FileExists(szStandalone))
964 if (!EraseFile(szStandalone))
965 LogSilentF("Network: Could not delete temporary resource file (%s)", strerror(errno));
966 szFile[0] = szStandalone[0] = '\0';
967 fDirty = false;
968 fTempFile = false;
969 Core.Clear();
970 Chunks.Clear();
971 fRemoved = false;
972 ClearLoad();
973 }
974
OpenFileRead()975 int32_t C4Network2Res::OpenFileRead()
976 {
977 CStdLock FileLock(&FileCSec);
978 if (!GetStandalone(nullptr, 0, false, false, true)) return -1;
979 // FIXME: Use standard OC file access api here
980 #ifdef _WIN32
981 return _wopen(GetWideChar(szStandalone), _O_BINARY | O_RDONLY);
982 #else
983 return open(szStandalone, _O_BINARY | O_CLOEXEC | O_RDONLY);
984 #endif
985 }
986
OpenFileWrite()987 int32_t C4Network2Res::OpenFileWrite()
988 {
989 CStdLock FileLock(&FileCSec);
990 // FIXME: Use standard OC file access api here
991 #ifdef _WIN32
992 return _wopen(GetWideChar(szStandalone), _O_BINARY | O_CREAT | O_WRONLY, S_IREAD | S_IWRITE);
993 #else
994 return open(szStandalone, _O_BINARY | O_CLOEXEC | O_CREAT | O_WRONLY, S_IREAD | S_IWRITE);
995 #endif
996 }
997
StartNewLoads()998 void C4Network2Res::StartNewLoads()
999 {
1000 if (!pCChunks) return;
1001 // count clients
1002 int32_t iCChunkCnt = 0; ClientChunks *pChunks;
1003 for (pChunks = pCChunks; pChunks; pChunks = pChunks->Next)
1004 iCChunkCnt++;
1005 // create array
1006 ClientChunks **pC = new ClientChunks *[iCChunkCnt];
1007 // initialize
1008 int32_t i;
1009 for (i = 0; i < iCChunkCnt; i++) pC[i] = nullptr;
1010 // create shuffled order
1011 for (pChunks = pCChunks, i = 0; i < iCChunkCnt; i++, pChunks = pChunks->Next)
1012 {
1013 // determine position
1014 int32_t iPos = UnsyncedRandom(iCChunkCnt - i);
1015 // find & set
1016 for (int32_t j = 0; ; j++)
1017 if (!pC[j] && !iPos--)
1018 {
1019 pC[j] = pChunks;
1020 break;
1021 }
1022 }
1023 // start new load until maximum count reached
1024 while (iLoadCnt + 1 <= C4NetResMaxLoad)
1025 {
1026 int32_t ioLoadCnt = iLoadCnt;
1027 // search someone
1028 for (i = 0; i < iCChunkCnt; i++)
1029 if (pC[i])
1030 {
1031 // try to start load
1032 if (!StartLoad(pC[i]->ClientID, pC[i]->Chunks))
1033 { RemoveCChunks(pC[i]); pC[i] = nullptr; continue; }
1034 // success?
1035 if (iLoadCnt > ioLoadCnt) break;
1036 }
1037 // not found?
1038 if (i >= iCChunkCnt)
1039 break;
1040 }
1041 // clear up
1042 delete [] pC;
1043 }
1044
StartLoad(int32_t iFromClient,const C4Network2ResChunkData & Available)1045 bool C4Network2Res::StartLoad(int32_t iFromClient, const C4Network2ResChunkData &Available)
1046 {
1047 assert(pParent && pParent->getIOClass());
1048 // all slots used? ignore
1049 if (iLoadCnt + 1 >= C4NetResMaxLoad) return true;
1050 // is there already a load by this client? ignore
1051 for (C4Network2ResLoad *pPos = pLoads; pPos; pPos = pPos->Next())
1052 if (pPos->getByClient() == iFromClient)
1053 return true;
1054 // find chunk to retrieve
1055 int32_t iLoads[C4NetResMaxLoad]; int32_t i = 0;
1056 for (C4Network2ResLoad *pLoad = pLoads; pLoad; pLoad = pLoad->Next())
1057 iLoads[i++] = pLoad->getChunk();
1058 int32_t iRetrieveChunk = Chunks.GetChunkToRetrieve(Available, i, iLoads);
1059 // nothing? ignore
1060 if (iRetrieveChunk < 0 || (uint32_t)iRetrieveChunk >= Core.getChunkCnt())
1061 return true;
1062 // search message connection for client
1063 C4Network2IOConnection *pConn = pParent->getIOClass()->GetMsgConnection(iFromClient);
1064 if (!pConn) return false;
1065 // send request
1066 if (!pConn->Send(MkC4NetIOPacket(PID_NetResReq, C4PacketResRequest(Core.getID(), iRetrieveChunk))))
1067 { pConn->DelRef(); return false; }
1068 pConn->DelRef();
1069 #ifdef C4NET2RES_DEBUG_LOG
1070 // log
1071 Application.InteractiveThread.ThreadLogS("Network: Res: requesting chunk %d of %d:%s (%s) from client %d",
1072 iRetrieveChunk, Core.getID(), Core.getFileName(), szFile, iFromClient);
1073 #endif
1074 // create load class
1075 C4Network2ResLoad *pnLoad = new C4Network2ResLoad(iRetrieveChunk, iFromClient);
1076 // add to list
1077 pnLoad->pNext = pLoads;
1078 pLoads = pnLoad;
1079 iLoadCnt++;
1080 // ok
1081 return true;
1082 }
1083
EndLoad()1084 void C4Network2Res::EndLoad()
1085 {
1086 // clear loading data
1087 ClearLoad();
1088 // set complete
1089 fLoading = false;
1090 // call handler
1091 assert(pParent);
1092 pParent->OnResComplete(this);
1093 }
1094
ClearLoad()1095 void C4Network2Res::ClearLoad()
1096 {
1097 // remove client chunks and loads
1098 fLoading = false;
1099 while (pCChunks) RemoveCChunks(pCChunks);
1100 while (pLoads) RemoveLoad(pLoads);
1101 iDiscoverStartTime = iLoadCnt = 0;
1102 }
1103
RemoveLoad(C4Network2ResLoad * pLoad)1104 void C4Network2Res::RemoveLoad(C4Network2ResLoad *pLoad)
1105 {
1106 if (pLoad == pLoads)
1107 pLoads = pLoad->Next();
1108 else
1109 {
1110 // find previous entry
1111 C4Network2ResLoad *pPrev;
1112 for (pPrev = pLoads; pPrev && pPrev->Next() != pLoad; pPrev = pPrev->Next()) {}
1113 // remove
1114 if (pPrev)
1115 pPrev->pNext = pLoad->Next();
1116 }
1117 // delete
1118 delete pLoad;
1119 iLoadCnt--;
1120 }
1121
RemoveCChunks(ClientChunks * pChunks)1122 void C4Network2Res::RemoveCChunks(ClientChunks *pChunks)
1123 {
1124 if (pChunks == pCChunks)
1125 pCChunks = pChunks->Next;
1126 else
1127 {
1128 // find previous entry
1129 ClientChunks *pPrev;
1130 for (pPrev = pCChunks; pPrev && pPrev->Next != pChunks; pPrev = pPrev->Next) {}
1131 // remove
1132 if (pPrev)
1133 pPrev->Next = pChunks->Next;
1134 }
1135 // delete
1136 delete pChunks;
1137 }
1138
OptimizeStandalone(bool fSilent)1139 bool C4Network2Res::OptimizeStandalone(bool fSilent)
1140 {
1141 CStdLock FileLock(&FileCSec);
1142 // for now: player files only
1143 if (Core.getType() == NRT_Player)
1144 {
1145 // log - this may take a few seconds
1146 if (!fSilent) LogF(LoadResStr("IDS_PRC_NETPREPARING"), GetFilename(szFile));
1147 // copy to temp file, if needed
1148 if (!fTempFile && SEqual(szFile, szStandalone))
1149 {
1150 char szNewStandalone[_MAX_PATH + 1];
1151 if (!pParent->FindTempResFileName(szStandalone, szNewStandalone))
1152 { if (!fSilent) Log("OptimizeStandalone: could not find free name for temporary file!"); return false; }
1153 if (!C4Group_CopyItem(szStandalone, szNewStandalone))
1154 { if (!fSilent) Log("OptimizeStandalone: could not copy to temporary file!"); return false; } /* TODO: Test failure */
1155 SCopy(szNewStandalone, szStandalone, sizeof(szStandalone) - 1);
1156 }
1157 // open as group
1158 C4Group Grp;
1159 if (!Grp.Open(szStandalone))
1160 { if (!fSilent) Log("OptimizeStandalone: could not open player file!"); return false; }
1161 // remove bigicon, if the file size is too large
1162 size_t iBigIconSize=0;
1163 if (Grp.FindEntry(C4CFN_BigIcon, nullptr, &iBigIconSize))
1164 if (iBigIconSize > C4NetResMaxBigicon*1024)
1165 Grp.Delete(C4CFN_BigIcon);
1166 Grp.Close();
1167 }
1168 return true;
1169 }
1170
1171 // *** C4Network2ResChunk
1172
1173 C4Network2ResChunk::C4Network2ResChunk() = default;
1174
1175 C4Network2ResChunk::~C4Network2ResChunk() = default;
1176
Set(C4Network2Res * pRes,uint32_t inChunk)1177 bool C4Network2ResChunk::Set(C4Network2Res *pRes, uint32_t inChunk)
1178 {
1179 const C4Network2ResCore &Core = pRes->getCore();
1180 iResID = pRes->getResID();
1181 iChunk = inChunk;
1182 // calculate offset and size
1183 int32_t iOffset = iChunk * Core.getChunkSize(),
1184 iSize = std::min<int32_t>(Core.getFileSize() - iOffset, C4NetResChunkSize);
1185 if (iSize < 0) { LogF("Network: could not get chunk from offset %d from resource file %s: File size is only %d!", iOffset, pRes->getFile(), Core.getFileSize()); return false; }
1186 // open file
1187 int32_t f = pRes->OpenFileRead();
1188 if (f == -1) { LogF("Network: could not open resource file %s!", pRes->getFile()); return false; }
1189 // seek
1190 if (iOffset)
1191 if (lseek(f, iOffset, SEEK_SET) != iOffset)
1192 { close(f); LogF("Network: could not read resource file %s!", pRes->getFile()); return false; }
1193 // read chunk of data
1194 char *pBuf = (char *) malloc(iSize);
1195 if (read(f, pBuf, iSize) != iSize)
1196 { free(pBuf); close(f); LogF("Network: could not read resource file %s!", pRes->getFile()); return false; }
1197 // set
1198 Data.Take(pBuf, iSize);
1199 // close
1200 close(f);
1201 // ok
1202 return true;
1203 }
1204
AddTo(C4Network2Res * pRes,C4Network2IO * pIO) const1205 bool C4Network2ResChunk::AddTo(C4Network2Res *pRes, C4Network2IO *pIO) const
1206 {
1207 assert(pRes); assert(pIO);
1208 const C4Network2ResCore &Core = pRes->getCore();
1209 // check
1210 if (iResID != pRes->getResID())
1211 {
1212 #ifdef C4NET2RES_DEBUG_LOG
1213 Application.InteractiveThread.ThreadLogS("C4Network2ResChunk(%d)::AddTo(%s [%d]): Resource ID mismatch!", (int) iResID, (const char *) Core.getFileName(), (int) pRes->getResID());
1214 #endif
1215 return false;
1216 }
1217 // calculate offset and size
1218 int32_t iOffset = iChunk * Core.getChunkSize();
1219 if (iOffset + Data.getSize() > Core.getFileSize())
1220 {
1221 #ifdef C4NET2RES_DEBUG_LOG
1222 Application.InteractiveThread.ThreadLogS("C4Network2ResChunk(%d)::AddTo(%s [%d]): Adding %d bytes at offset %d exceeds expected file size of %d!", (int) iResID, (const char *) Core.getFileName(), (int) pRes->getResID(), (int) Data.getSize(), (int) iOffset, (int) Core.getFileSize());
1223 #endif
1224 return false;
1225 }
1226 // open file
1227 int32_t f = pRes->OpenFileWrite();
1228 if (f == -1)
1229 {
1230 #ifdef C4NET2RES_DEBUG_LOG
1231 Application.InteractiveThread.ThreadLogS("C4Network2ResChunk(%d)::AddTo(%s [%d]): Open write file error: %s!", (int) iResID, (const char *) Core.getFileName(), (int) pRes->getResID(), strerror(errno));
1232 #endif
1233 return false;
1234 }
1235 // seek
1236 if (iOffset)
1237 if (lseek(f, iOffset, SEEK_SET) != iOffset)
1238 {
1239 #ifdef C4NET2RES_DEBUG_LOG
1240 Application.InteractiveThread.ThreadLogS("C4Network2ResChunk(%d)::AddTo(%s [%d]): lseek file error: %s!", (int) iResID, (const char *) Core.getFileName(), (int) pRes->getResID(), strerror(errno));
1241 #endif
1242 close(f);
1243 return false;
1244 }
1245 // write
1246 if (write(f, Data.getData(), Data.getSize()) != int32_t(Data.getSize()))
1247 {
1248 #ifdef C4NET2RES_DEBUG_LOG
1249 Application.InteractiveThread.ThreadLogS("C4Network2ResChunk(%d)::AddTo(%s [%d]): write error: %s!", (int) iResID, (const char *) Core.getFileName(), (int) pRes->getResID(), strerror(errno));
1250 #endif
1251 close(f);
1252 return false;
1253 }
1254 // ok, add chunks
1255 close(f);
1256 pRes->Chunks.AddChunk(iChunk);
1257 return true;
1258 }
1259
CompileFunc(StdCompiler * pComp)1260 void C4Network2ResChunk::CompileFunc(StdCompiler *pComp)
1261 {
1262 // pack header
1263 pComp->Value(mkNamingAdapt(iResID, "ResID", -1));
1264 pComp->Value(mkNamingAdapt(iChunk, "Chunk", ~0U));
1265 // Data
1266 pComp->Value(mkNamingAdapt(Data, "Data"));
1267 }
1268
1269 // *** C4Network2ResList
1270
C4Network2ResList()1271 C4Network2ResList::C4Network2ResList()
1272 : ResListCSec(this),
1273 iNextResID((-1) << 16)
1274 {
1275
1276 }
1277
~C4Network2ResList()1278 C4Network2ResList::~C4Network2ResList()
1279 {
1280 Clear();
1281 }
1282
Init(int32_t inClientID,C4Network2IO * pIOClass)1283 bool C4Network2ResList::Init(int32_t inClientID, C4Network2IO *pIOClass) // by main thread
1284 {
1285 // clear old list
1286 Clear();
1287 // safe IO class
1288 pIO = pIOClass;
1289 // set client id
1290 iNextResID = iClientID = 0;
1291 SetLocalID(inClientID);
1292 // create network path
1293 if (!CreateNetworkFolder()) return false;
1294 // ok
1295 return true;
1296 }
1297
SetLocalID(int32_t inClientID)1298 void C4Network2ResList::SetLocalID(int32_t inClientID)
1299 {
1300 CStdLock ResIDLock(&ResIDCSec);
1301 int32_t iOldClientID = iClientID;
1302 int32_t iIDDiff = (inClientID - iClientID) << 16;
1303 // set new counter
1304 iClientID = inClientID;
1305 iNextResID += iIDDiff;
1306 // change resource ids
1307 CStdLock ResListLock(&ResListCSec);
1308 for (C4Network2Res *pRes = pFirst; pRes; pRes = pRes->pNext)
1309 if (pRes->getResClient() == iOldClientID)
1310 pRes->ChangeID(pRes->getResID() + iIDDiff);
1311 }
1312
nextResID()1313 int32_t C4Network2ResList::nextResID() // by main thread
1314 {
1315 CStdLock ResIDLock(&ResIDCSec);
1316 assert(iNextResID >= (iClientID << 16));
1317 if (iNextResID >= ((iClientID+1) << 16) - 1)
1318 iNextResID = std::max<int32_t>(0, iClientID) << 16;
1319 // find free
1320 while (getRes(iNextResID))
1321 iNextResID++;
1322 return iNextResID++;
1323 }
1324
getRes(int32_t iResID)1325 C4Network2Res *C4Network2ResList::getRes(int32_t iResID)
1326 {
1327 CStdShareLock ResListLock(&ResListCSec);
1328 for (C4Network2Res *pCur = pFirst; pCur; pCur = pCur->pNext)
1329 if (pCur->getResID() == iResID)
1330 return pCur;
1331 return nullptr;
1332 }
1333
getRes(const char * szFile,bool fLocalOnly)1334 C4Network2Res *C4Network2ResList::getRes(const char *szFile, bool fLocalOnly)
1335 {
1336 CStdShareLock ResListLock(&ResListCSec);
1337 for (C4Network2Res *pCur = pFirst; pCur; pCur = pCur->pNext)
1338 if (!pCur->isAnonymous())
1339 if (SEqual(pCur->getFile(), szFile))
1340 if (!fLocalOnly || pCur->getResClient()==iClientID)
1341 return pCur;
1342 return nullptr;
1343 }
1344
getRefRes(int32_t iResID)1345 C4Network2Res::Ref C4Network2ResList::getRefRes(int32_t iResID)
1346 {
1347 CStdShareLock ResListLock(&ResListCSec);
1348 return getRes(iResID);
1349 }
1350
getRefRes(const char * szFile,bool fLocalOnly)1351 C4Network2Res::Ref C4Network2ResList::getRefRes(const char *szFile, bool fLocalOnly)
1352 {
1353 CStdShareLock ResListLock(&ResListCSec);
1354 return getRes(szFile, fLocalOnly);
1355 }
1356
getRefNextRes(int32_t iResID)1357 C4Network2Res::Ref C4Network2ResList::getRefNextRes(int32_t iResID)
1358 {
1359 CStdShareLock ResListLock(&ResListCSec);
1360 C4Network2Res *pRes = nullptr;
1361 for (C4Network2Res *pCur = pFirst; pCur; pCur = pCur->pNext)
1362 if (!pCur->isRemoved() && pCur->getResID() >= iResID)
1363 if (!pRes || pRes->getResID() > pCur->getResID())
1364 pRes = pCur;
1365 return pRes;
1366 }
1367
Add(C4Network2Res * pRes)1368 void C4Network2ResList::Add(C4Network2Res *pRes)
1369 {
1370 // get locks
1371 CStdShareLock ResListLock(&ResListCSec);
1372 CStdLock ResListAddLock(&ResListAddCSec);
1373 // reference
1374 pRes->AddRef();
1375 // add
1376 pRes->pNext = pFirst;
1377 pFirst = pRes;
1378 }
1379
AddByFile(const char * strFilePath,bool fTemp,C4Network2ResType eType,int32_t iResID,const char * szResName,bool fAllowUnloadable)1380 C4Network2Res::Ref C4Network2ResList::AddByFile(const char *strFilePath, bool fTemp, C4Network2ResType eType, int32_t iResID, const char *szResName, bool fAllowUnloadable)
1381 {
1382 // already in list?
1383 C4Network2Res::Ref pRes = getRefRes(strFilePath);
1384 if (pRes) return pRes;
1385 // get resource ID
1386 if (iResID < 0) iResID = nextResID();
1387 if (iResID < 0) { Log("AddByFile: no more resource IDs available!"); return nullptr; }
1388 // create new
1389 pRes = new C4Network2Res(this);
1390 // initialize
1391 if (!pRes->SetByFile(strFilePath, fTemp, eType, iResID, szResName)) { return nullptr; }
1392 // create standalone for non-system files
1393 // system files shouldn't create a standalone; they should never be marked loadable!
1394 if (eType != NRT_System)
1395 if (!pRes->GetStandalone(nullptr, 0, true, fAllowUnloadable))
1396 if (!fAllowUnloadable)
1397 {
1398 delete pRes;
1399 return nullptr;
1400 }
1401 // add to list
1402 Add(pRes);
1403 return pRes;
1404 }
1405
AddByGroup(C4Group * pGrp,bool fTemp,C4Network2ResType eType,int32_t iResID,const char * szResName,bool fAllowUnloadable)1406 C4Network2Res::Ref C4Network2ResList::AddByGroup(C4Group *pGrp, bool fTemp, C4Network2ResType eType, int32_t iResID, const char *szResName, bool fAllowUnloadable)
1407 {
1408 // get resource ID
1409 if (iResID < 0) iResID = nextResID();
1410 if (iResID < 0) { Log("AddByGroup: no more resource IDs available!"); return nullptr; }
1411 // create new
1412 C4Network2Res::Ref pRes = new C4Network2Res(this);
1413 // initialize
1414 if (!pRes->SetByGroup(pGrp, fTemp, eType, iResID, szResName))
1415 {
1416 delete pRes;
1417 return nullptr;
1418 }
1419 // create standalone
1420 if (!pRes->GetStandalone(nullptr, 0, true, fAllowUnloadable))
1421 if (!fAllowUnloadable)
1422 {
1423 delete pRes;
1424 return nullptr;
1425 }
1426 // add to list
1427 Add(pRes);
1428 return pRes;
1429 }
1430
AddByCore(const C4Network2ResCore & Core,bool fLoad)1431 C4Network2Res::Ref C4Network2ResList::AddByCore(const C4Network2ResCore &Core, bool fLoad) // by main thread
1432 {
1433 // already in list?
1434 C4Network2Res::Ref pRes = getRefRes(Core.getID());
1435 if (pRes) return pRes;
1436 #ifdef C4NET2RES_LOAD_ALL
1437 // load without check (if possible)
1438 if (Core.isLoadable()) return AddLoad(Core);
1439 #endif
1440 // create new
1441 pRes = new C4Network2Res(this);
1442 // try set by core
1443 if (!pRes->SetByCore(Core, true))
1444 {
1445 pRes.Clear();
1446 // try load (if specified)
1447 return fLoad ? AddLoad(Core) : nullptr;
1448 }
1449 // log
1450 Application.InteractiveThread.ThreadLogS("Network: Found identical %s. Not loading.", pRes->getCore().getFileName());
1451 // add to list
1452 Add(pRes);
1453 // ok
1454 return pRes;
1455 }
1456
AddLoad(const C4Network2ResCore & Core)1457 C4Network2Res::Ref C4Network2ResList::AddLoad(const C4Network2ResCore &Core) // by main thread
1458 {
1459 // marked unloadable by creator?
1460 if (!Core.isLoadable())
1461 {
1462 // show error msg
1463 Application.InteractiveThread.ThreadLog("Network: Cannot load %s (marked unloadable)", Core.getFileName());
1464 return nullptr;
1465 }
1466 // create new
1467 C4Network2Res::Ref pRes = new C4Network2Res(this);
1468 // initialize
1469 pRes->SetLoad(Core);
1470 // log
1471 Application.InteractiveThread.ThreadLogS("Network: loading %s...", Core.getFileName());
1472 // add to list
1473 Add(pRes);
1474 return pRes;
1475 }
1476
RemoveAtClient(int32_t iClientID)1477 void C4Network2ResList::RemoveAtClient(int32_t iClientID) // by main thread
1478 {
1479 CStdShareLock ResListLock(&ResListCSec);
1480 for (C4Network2Res *pRes = pFirst; pRes; pRes = pRes->pNext)
1481 if (pRes->getResClient() == iClientID)
1482 pRes->Remove();
1483 }
1484
Clear()1485 void C4Network2ResList::Clear()
1486 {
1487 CStdShareLock ResListLock(&ResListCSec);
1488 for (C4Network2Res *pRes = pFirst; pRes; pRes = pRes->pNext)
1489 {
1490 pRes->Remove();
1491 pRes->iLastReqTime = 0;
1492 }
1493 iClientID = C4ClientIDUnknown;
1494 iLastDiscover = iLastStatus = 0;
1495 }
1496
OnClientConnect(C4Network2IOConnection * pConn)1497 void C4Network2ResList::OnClientConnect(C4Network2IOConnection *pConn) // by main thread
1498 {
1499 // discover resources
1500 SendDiscover(pConn);
1501 }
1502
HandlePacket(char cStatus,const C4PacketBase * pPacket,C4Network2IOConnection * pConn)1503 void C4Network2ResList::HandlePacket(char cStatus, const C4PacketBase *pPacket, C4Network2IOConnection *pConn)
1504 {
1505 // security
1506 if (!pConn) return;
1507
1508 #define GETPKT(type, name) \
1509 assert(pPacket); const type &name = \
1510 static_cast<const type &>(*pPacket);
1511
1512 switch (cStatus)
1513 {
1514
1515 case PID_NetResDis: // resource discover
1516 {
1517 if (!pConn->isOpen()) break;
1518 GETPKT(C4PacketResDiscover, Pkt);
1519 // search matching resources
1520 CStdShareLock ResListLock(&ResListCSec);
1521 for (C4Network2Res *pRes = pFirst; pRes; pRes = pRes->pNext)
1522 if (Pkt.isIDPresent(pRes->getResID()))
1523 // must be binary compatible
1524 if (pRes->IsBinaryCompatible())
1525 pRes->OnDiscover(pConn);
1526 }
1527 break;
1528
1529 case PID_NetResStat: // resource status
1530 {
1531 if (!pConn->isOpen()) break;
1532 GETPKT(C4PacketResStatus, Pkt);
1533 // matching resource?
1534 CStdShareLock ResListLock(&ResListCSec);
1535 C4Network2Res *pRes = getRes(Pkt.getResID());
1536 // present / being loaded? call handler
1537 if (pRes)
1538 pRes->OnStatus(Pkt.getChunks(), pConn);
1539 }
1540 break;
1541
1542 case PID_NetResDerive: // resource derive
1543 {
1544 GETPKT(C4Network2ResCore, Core);
1545 if (Core.getDerID() < 0) break;
1546 // Check if there is a anonymous derived resource with matching parent.
1547 CStdShareLock ResListLock(&ResListCSec);
1548 for (C4Network2Res *pRes = pFirst; pRes; pRes = pRes->pNext)
1549 if (pRes->isAnonymous() && pRes->getCore().getDerID() == Core.getDerID())
1550 pRes->FinishDerive(Core);
1551 }
1552 break;
1553
1554 case PID_NetResReq: // resource request
1555 {
1556 GETPKT(C4PacketResRequest, Pkt);
1557 // find resource
1558 CStdShareLock ResListLock(&ResListCSec);
1559 C4Network2Res *pRes = getRes(Pkt.getReqID());
1560 // send requested chunk
1561 if (pRes && pRes->IsBinaryCompatible()) pRes->SendChunk(Pkt.getReqChunk(), pConn->getClientID());
1562 }
1563 break;
1564
1565 case PID_NetResData: // a chunk of data is coming in
1566 {
1567 GETPKT(C4Network2ResChunk, Chunk);
1568 // find resource
1569 CStdShareLock ResListLock(&ResListCSec);
1570 C4Network2Res *pRes = getRes(Chunk.getResID());
1571 // send requested chunk
1572 if (pRes) pRes->OnChunk(Chunk);
1573 }
1574 break;
1575 }
1576 #undef GETPKT
1577 }
1578
OnTimer()1579 void C4Network2ResList::OnTimer()
1580 {
1581 CStdShareLock ResListLock(&ResListCSec);
1582 C4Network2Res *pRes;
1583 // do loads, check timeouts
1584 for (pRes = pFirst; pRes; pRes = pRes->pNext)
1585 if (pRes->isLoading() && !pRes->isRemoved())
1586 if (!pRes->DoLoad())
1587 pRes->Remove();
1588 // discovery time?
1589 if (!iLastDiscover || difftime(time(nullptr), iLastDiscover) >= C4NetResDiscoverInterval)
1590 {
1591 // needed?
1592 bool fSendDiscover = false;
1593 for (C4Network2Res *pRes = pFirst; pRes; pRes = pRes->pNext)
1594 if (pRes->isLoading() && !pRes->isRemoved())
1595 fSendDiscover |= pRes->NeedsDiscover();
1596 // send
1597 if (fSendDiscover)
1598 SendDiscover();
1599 }
1600 // status update?
1601 if (!iLastStatus || difftime(time(nullptr), iLastStatus) >= C4NetResStatusInterval)
1602 {
1603 // any?
1604 bool fStatusUpdates = false;
1605 for (pRes = pFirst; pRes; pRes = pRes->pNext)
1606 if (pRes->isDirty() && !pRes->isRemoved())
1607 fStatusUpdates |= pRes->SendStatus();
1608 // set time accordingly
1609 iLastStatus = fStatusUpdates ? time(nullptr) : 0;
1610 }
1611 }
1612
OnShareFree(CStdCSecEx * pCSec)1613 void C4Network2ResList::OnShareFree(CStdCSecEx *pCSec)
1614 {
1615 if (pCSec == &ResListCSec)
1616 {
1617 // remove entries
1618 for (C4Network2Res *pRes = pFirst, *pNext, *pPrev = nullptr; pRes; pRes = pNext)
1619 {
1620 pNext = pRes->pNext;
1621 if (pRes->isRemoved() && (!pRes->getLastReqTime() || difftime(time(nullptr), pRes->getLastReqTime()) > C4NetResDeleteTime))
1622 {
1623 // unlink
1624 (pPrev ? pPrev->pNext : pFirst) = pNext;
1625 // remove
1626 pRes->pNext = nullptr;
1627 pRes->DelRef();
1628 }
1629 else
1630 pPrev = pRes;
1631 }
1632 }
1633 }
1634
SendDiscover(C4Network2IOConnection * pTo)1635 bool C4Network2ResList::SendDiscover(C4Network2IOConnection *pTo) // by both
1636 {
1637 // make packet
1638 C4PacketResDiscover Pkt;
1639 // add special retrieves
1640 CStdShareLock ResListLock(&ResListCSec);
1641 for (C4Network2Res *pRes = pFirst; pRes; pRes = pRes->pNext)
1642 if (!pRes->isRemoved())
1643 if (pRes->isLoading())
1644 Pkt.AddDisID(pRes->getResID());
1645 ResListLock.Clear();
1646 // empty?
1647 if (!Pkt.getDisIDCnt()) return false;
1648 // broadcast?
1649 if (!pTo)
1650 {
1651 // save time
1652 iLastDiscover = time(nullptr);
1653 // send
1654 return pIO->BroadcastMsg(MkC4NetIOPacket(PID_NetResDis, Pkt));
1655 }
1656 else
1657 return pTo->Send(MkC4NetIOPacket(PID_NetResDis, Pkt));
1658 }
1659
OnResComplete(C4Network2Res * pRes)1660 void C4Network2ResList::OnResComplete(C4Network2Res *pRes)
1661 {
1662 // log (network thread -> ThreadLog)
1663 Application.InteractiveThread.ThreadLogS("Network: %s received.", pRes->getCore().getFileName());
1664 // call handler (ctrl might wait for this resource)
1665 ::Control.Network.OnResComplete(pRes);
1666 }
1667
CreateNetworkFolder()1668 bool C4Network2ResList::CreateNetworkFolder()
1669 {
1670 // get network path without trailing backslash
1671 char szNetworkPath[_MAX_PATH+1];
1672 SCopy(Config.AtNetworkPath(""), szNetworkPath, _MAX_PATH);
1673 TruncateBackslash(szNetworkPath);
1674 // but make sure that the configured path has one
1675 AppendBackslash(Config.Network.WorkPath);
1676 // does not exist?
1677 if (!DirectoryExists(szNetworkPath))
1678 {
1679 if (!CreatePath(szNetworkPath))
1680 { LogFatal("Network: could not create network path!"); return false; }
1681 return true;
1682 }
1683 return true;
1684 }
1685
FindTempResFileName(const char * szFilename,char * pTarget)1686 bool C4Network2ResList::FindTempResFileName(const char *szFilename, char *pTarget)
1687 {
1688 char safeFilename[_MAX_PATH];
1689 char* safePos = safeFilename;
1690 while (*szFilename)
1691 {
1692 if ((*szFilename >= 'a' && *szFilename <= 'z') ||
1693 (*szFilename >= 'A' && *szFilename <= 'Z') ||
1694 (*szFilename >= '0' && *szFilename <= '9') ||
1695 (*szFilename == '.') || (*szFilename == '/'))
1696 *safePos = *szFilename;
1697 else
1698 *safePos = '_';
1699
1700 ++safePos;
1701 ++szFilename;
1702 }
1703 *safePos = 0;
1704 szFilename = safeFilename;
1705
1706 // create temporary file
1707 SCopy(Config.AtNetworkPath(GetFilename(szFilename)), pTarget, _MAX_PATH);
1708 // file name is free?
1709 if (!ItemExists(pTarget)) return true;
1710 // find another file name
1711 char szFileMask[_MAX_PATH+1];
1712 SCopy(pTarget, szFileMask, GetExtension(pTarget)-1-pTarget);
1713 SAppend("_%d", szFileMask, _MAX_PATH);
1714 SAppend(GetExtension(pTarget)-1, szFileMask, _MAX_PATH);
1715 for (int32_t i = 2; i < 1000; i++)
1716 {
1717 snprintf(pTarget, _MAX_PATH, szFileMask, i);
1718 // doesn't exist?
1719 if (!ItemExists(pTarget))
1720 return true;
1721 }
1722 // not found
1723 return false;
1724 }
1725