1 /*
2 * serialization_utils.h
3 * ---------------------
4 * Purpose: Serializing data to and from MPTM files.
5 * Notes : (currently none)
6 * Authors: OpenMPT Devs
7 * The OpenMPT source code is released under the BSD license. Read LICENSE for more details.
8 */
9
10
11 #pragma once
12
13 #include "openmpt/all/BuildSettings.hpp"
14
15 #include "mpt/io/io.hpp"
16 #include "mpt/io/io_stdstream.hpp"
17 #include "openmpt/base/Endian.hpp"
18
19 #include "../common/mptBaseTypes.h"
20
21 #include <algorithm>
22 #include <bitset>
23 #include <ios>
24 #include <iosfwd>
25 #include <limits>
26 #include <string>
27 #include <vector>
28
29 #include <istream>
30 #include <ostream>
31
32 #include <cstring>
33
34 OPENMPT_NAMESPACE_BEGIN
35
36 namespace srlztn //SeRiaLiZaTioN
37 {
38
39 typedef std::ios::off_type Offtype;
40 typedef Offtype Postype;
41
42 typedef uintptr_t DataSize; // Data size type.
43 typedef uintptr_t RposType; // Relative position type.
44 typedef uintptr_t NumType; // Entry count type.
45
46 const DataSize invalidDatasize = DataSize(-1);
47
48 enum
49 {
50 SNT_PROGRESS = 0x08000000, // = 1 << 27
51 SNT_FAILURE = 0x40000000, // = 1 << 30
52 SNT_NOTE = 0x20000000, // = 1 << 29
53 SNT_WARNING = 0x10000000, // = 1 << 28
54 SNT_NONE = 0,
55
56 SNRW_BADGIVEN_STREAM = 1 | SNT_FAILURE,
57
58 // Read failures.
59 SNR_BADSTREAM_AFTER_MAPHEADERSEEK = 2 | SNT_FAILURE,
60 SNR_STARTBYTE_MISMATCH = 3 | SNT_FAILURE,
61 SNR_BADSTREAM_AT_MAP_READ = 4 | SNT_FAILURE,
62 SNR_INSUFFICIENT_STREAM_OFFTYPE = 5 | SNT_FAILURE,
63 SNR_OBJECTCLASS_IDMISMATCH = 6 | SNT_FAILURE,
64 SNR_TOO_MANY_ENTRIES_TO_READ = 7 | SNT_FAILURE,
65 SNR_INSUFFICIENT_RPOSTYPE = 8 | SNT_FAILURE,
66
67 // Read notes and warnings.
68 SNR_ZEROENTRYCOUNT = 0x80 | SNT_NOTE, // 0x80 == 1 << 7
69 SNR_NO_ENTRYIDS_WITH_CUSTOMID_DEFINED = 0x100 | SNT_NOTE,
70 SNR_LOADING_OBJECT_WITH_LARGER_VERSION = 0x200 | SNT_NOTE,
71
72 // Write failures.
73 SNW_INSUFFICIENT_FIXEDSIZE = (0x10) | SNT_FAILURE,
74 SNW_CHANGING_IDSIZE_WITH_FIXED_IDSIZESETTING = (0x11) | SNT_FAILURE,
75 SNW_DATASIZETYPE_OVERFLOW = (0x13) | SNT_FAILURE,
76 SNW_MAX_WRITE_COUNT_REACHED = (0x14) | SNT_FAILURE,
77 SNW_INSUFFICIENT_DATASIZETYPE = (0x16) | SNT_FAILURE
78 };
79
80
81 enum
82 {
83 IdSizeVariable = std::numeric_limits<uint16>::max(),
84 IdSizeMaxFixedSize = (std::numeric_limits<uint8>::max() >> 1)
85 };
86
87 typedef int32 SsbStatus;
88
89
90 struct ReadEntry
91 {
ReadEntryReadEntry92 ReadEntry() : nIdpos(0), rposStart(0), nSize(invalidDatasize), nIdLength(0) {}
93
94 uintptr_t nIdpos; // Index of id start in ID array.
95 RposType rposStart; // Entry start position.
96 DataSize nSize; // Entry size.
97 uint16 nIdLength; // Length of id.
98 };
99
100
101 enum Rwf
102 {
103 RwfWMapStartPosEntry, // Write. True to include data start pos entry to map.
104 RwfWMapSizeEntry, // Write. True to include data size entry to map.
105 RwfWMapDescEntry, // Write. True to include description entry to map.
106 RwfWVersionNum, // Write. True to include version numeric.
107 RwfRMapCached, // Read. True if map has been cached.
108 RwfRMapHasId, // Read. True if map has IDs
109 RwfRMapHasStartpos, // Read. True if map data start pos.
110 RwfRMapHasSize, // Read. True if map has entry size.
111 RwfRMapHasDesc, // Read. True if map has entry description.
112 RwfRTwoBytesDescChar, // Read. True if map description characters are two bytes.
113 RwfRHeaderIsRead, // Read. True when header is read.
114 RwfRwHasMap, // Read/write. True if map exists.
115 RwfNumFlags
116 };
117
118
119 template<class T>
Binarywrite(std::ostream & oStrm,const T & data)120 inline void Binarywrite(std::ostream& oStrm, const T& data)
121 {
122 mpt::IO::WriteIntLE(oStrm, data);
123 }
124
125 template<>
Binarywrite(std::ostream & oStrm,const float & data)126 inline void Binarywrite(std::ostream& oStrm, const float& data)
127 {
128 IEEE754binary32LE tmp = IEEE754binary32LE(data);
129 mpt::IO::Write(oStrm, tmp);
130 }
131
132 template<>
Binarywrite(std::ostream & oStrm,const double & data)133 inline void Binarywrite(std::ostream& oStrm, const double& data)
134 {
135 IEEE754binary64LE tmp = IEEE754binary64LE(data);
136 mpt::IO::Write(oStrm, tmp);
137 }
138
139 template <class T>
WriteItem(std::ostream & oStrm,const T & data)140 inline void WriteItem(std::ostream& oStrm, const T& data)
141 {
142 static_assert(std::is_trivial<T>::value == true, "");
143 Binarywrite(oStrm, data);
144 }
145
146 void WriteItemString(std::ostream& oStrm, const std::string &str);
147
148 template <>
149 inline void WriteItem<std::string>(std::ostream& oStrm, const std::string& str) {WriteItemString(oStrm, str);}
150
151
152 template<class T>
Binaryread(std::istream & iStrm,T & data)153 inline void Binaryread(std::istream& iStrm, T& data)
154 {
155 mpt::IO::ReadIntLE(iStrm, data);
156 }
157
158 template<>
Binaryread(std::istream & iStrm,float & data)159 inline void Binaryread(std::istream& iStrm, float& data)
160 {
161 IEEE754binary32LE tmp = IEEE754binary32LE(0.0f);
162 mpt::IO::Read(iStrm, tmp);
163 data = tmp;
164 }
165
166 template<>
Binaryread(std::istream & iStrm,double & data)167 inline void Binaryread(std::istream& iStrm, double& data)
168 {
169 IEEE754binary64LE tmp = IEEE754binary64LE(0.0);
170 mpt::IO::Read(iStrm, tmp);
171 data = tmp;
172 }
173
174 //Read only given number of bytes to the beginning of data; data bytes are memset to 0 before reading.
175 template <class T>
Binaryread(std::istream & iStrm,T & data,const Offtype bytecount)176 inline void Binaryread(std::istream& iStrm, T& data, const Offtype bytecount)
177 {
178 mpt::IO::ReadBinaryTruncatedLE(iStrm, data, static_cast<std::size_t>(bytecount));
179 }
180
181 template <>
182 inline void Binaryread<float>(std::istream& iStrm, float& data, const Offtype bytecount)
183 {
184 typedef IEEE754binary32LE T;
185 std::byte bytes[sizeof(T)];
186 std::memset(bytes, 0, sizeof(T));
187 mpt::IO::ReadRaw(iStrm, bytes, std::min(static_cast<std::size_t>(bytecount), sizeof(T)));
188 // There is not much we can sanely do for truncated floats,
189 // thus we ignore what we just read and return 0.
190 data = 0.0f;
191 }
192
193 template <>
194 inline void Binaryread<double>(std::istream& iStrm, double& data, const Offtype bytecount)
195 {
196 typedef IEEE754binary64LE T;
197 std::byte bytes[sizeof(T)];
198 std::memset(bytes, 0, sizeof(T));
199 mpt::IO::ReadRaw(iStrm, bytes, std::min(static_cast<std::size_t>(bytecount), sizeof(T)));
200 // There is not much we can sanely do for truncated floats,
201 // thus we ignore what we just read and return 0.
202 data = 0.0;
203 }
204
205
206 template <class T>
ReadItem(std::istream & iStrm,T & data,const DataSize nSize)207 inline void ReadItem(std::istream& iStrm, T& data, const DataSize nSize)
208 {
209 static_assert(std::is_trivial<T>::value == true, "");
210 if (nSize == sizeof(T) || nSize == invalidDatasize)
211 Binaryread(iStrm, data);
212 else
213 Binaryread(iStrm, data, nSize);
214 }
215
216 void ReadItemString(std::istream& iStrm, std::string& str, const DataSize);
217
218 template <>
219 inline void ReadItem<std::string>(std::istream& iStrm, std::string& str, const DataSize nSize)
220 {
221 ReadItemString(iStrm, str, nSize);
222 }
223
224
225
226 class ID
227 {
228 private:
229 std::string m_ID; // NOTE: can contain null characters ('\0')
230 public:
ID()231 ID() { }
ID(const std::string & id)232 ID(const std::string &id) : m_ID(id) { }
ID(const char * beg,const char * end)233 ID(const char *beg, const char *end) : m_ID(beg, end) { }
ID(const char * id)234 ID(const char *id) : m_ID(id?id:"") { }
ID(const char * str,std::size_t len)235 ID(const char * str, std::size_t len) : m_ID(str, str + len) { }
236 template <typename T>
FromInt(const T & val)237 static ID FromInt(const T &val)
238 {
239 static_assert(std::numeric_limits<T>::is_integer);
240 typename mpt::make_le<T>::type valle;
241 valle = val;
242 return ID(std::string(mpt::byte_cast<const char*>(mpt::as_raw_memory(valle).data()), mpt::byte_cast<const char*>(mpt::as_raw_memory(valle).data() + sizeof(valle))));
243 }
244 bool IsPrintable() const;
245 mpt::ustring AsString() const;
GetBytes()246 const char *GetBytes() const { return m_ID.c_str(); }
GetSize()247 std::size_t GetSize() const { return m_ID.length(); }
248 bool operator == (const ID &other) const { return m_ID == other.m_ID; }
249 bool operator != (const ID &other) const { return m_ID != other.m_ID; }
250 };
251
252
253
254 class Ssb
255 {
256
257 protected:
258
259 Ssb();
260
261 public:
262
GetStatus()263 SsbStatus GetStatus() const
264 {
265 return m_Status;
266 }
267
268 protected:
269
270 // When writing, returns the number of entries written.
271 // When reading, returns the number of entries read not including unrecognized entries.
GetCounter()272 NumType GetCounter() const {return m_nCounter;}
273
SetFlag(Rwf flag,bool val)274 void SetFlag(Rwf flag, bool val) {m_Flags.set(flag, val);}
GetFlag(Rwf flag)275 bool GetFlag(Rwf flag) const {return m_Flags[flag];}
276
277 protected:
278
279 SsbStatus m_Status;
280
281 uint32 m_nFixedEntrySize; // Read/write: If > 0, data entries have given fixed size.
282
283 Postype m_posStart; // Read/write: Stream position at the beginning of object.
284
285 uint16 m_nIdbytes; // Read/Write: Tells map ID entry size in bytes. If size is variable, value is IdSizeVariable.
286 NumType m_nCounter; // Read/write: Keeps count of entries written/read.
287
288 std::bitset<RwfNumFlags> m_Flags; // Read/write: Various flags.
289
290 protected:
291
292 enum : uint8 { s_DefaultFlagbyte = 0 };
293 static const char s_EntryID[3];
294
295 };
296
297
298
299 class SsbRead
300 : public Ssb
301 {
302
303 public:
304
305 enum ReadRv // Read return value.
306 {
307 EntryRead,
308 EntryNotFound
309 };
310 enum IdMatchStatus
311 {
312 IdMatch, IdMismatch
313 };
314 typedef std::vector<ReadEntry>::const_iterator ReadIterator;
315
316 SsbRead(std::istream& iStrm);
317
318 // Call this to begin reading: must be called before other read functions.
319 void BeginRead(const ID &id, const uint64& nVersion);
320
321 // After calling BeginRead(), this returns number of entries in the file.
GetNumEntries()322 NumType GetNumEntries() const {return m_nReadEntrycount;}
323
324 // Returns read iterator to the beginning of entries.
325 // The behaviour of read iterators is undefined if map doesn't
326 // contain entry ids or data begin positions.
327 ReadIterator GetReadBegin();
328
329 // Returns read iterator to the end(one past last) of entries.
330 ReadIterator GetReadEnd();
331
332 // Compares given id with read entry id
333 IdMatchStatus CompareId(const ReadIterator& iter, const ID &id);
334
GetReadVersion()335 uint64 GetReadVersion() {return m_nReadVersion;}
336
337 // Read item using default read implementation.
338 template <class T>
ReadItem(T & obj,const ID & id)339 ReadRv ReadItem(T& obj, const ID &id) {return ReadItem(obj, id, srlztn::ReadItem<T>);}
340
341 // Read item using given function.
342 template <class T, class FuncObj>
343 ReadRv ReadItem(T& obj, const ID &id, FuncObj);
344
345 // Read item using read iterator.
346 template <class T>
ReadIterItem(const ReadIterator & iter,T & obj)347 ReadRv ReadIterItem(const ReadIterator& iter, T& obj) {return ReadIterItem(iter, obj, srlztn::ReadItem<T>);}
348 template <class T, class FuncObj>
349 ReadRv ReadIterItem(const ReadIterator& iter, T& obj, FuncObj func);
350
351 private:
352
353 // Reads map to cache.
354 void CacheMap();
355
356 // Searches for entry with given ID. If found, returns pointer to corresponding entry, else
357 // returns nullptr.
358 const ReadEntry* Find(const ID &id);
359
360 // Called after reading an object.
361 ReadRv OnReadEntry(const ReadEntry* pE, const ID &id, const Postype& posReadBegin);
362
363 void AddReadNote(const SsbStatus s);
364
365 // Called after reading entry. pRe is a pointer to associated map entry if exists.
366 void AddReadNote(const ReadEntry* const pRe, const NumType nNum);
367
368 void ResetReadstatus();
369
370 private:
371
372 // mapData is a cache that facilitates faster access to the stored data
373 // without having to reparse on every access.
374 // Iterator invalidation in CacheMap() is not a problem because every code
375 // path that ever returns an iterator into mapData does CacheMap exactly once
376 // beforehand. Following calls use this already cached map. As the data is
377 // immutable when reading, there is no need to ever invalidate the cache and
378 // redo CacheMap().
379
380 std::istream& iStrm;
381
382 std::vector<char> m_Idarray; // Read: Holds entry ids.
383
384 std::vector<ReadEntry> mapData; // Read: Contains map information.
385 uint64 m_nReadVersion; // Read: Version is placed here when reading.
386 RposType m_rposMapBegin; // Read: If map exists, rpos of map begin, else m_rposEndofHdrData.
387 Postype m_posMapEnd; // Read: If map exists, map end position, else pos of end of hdrData.
388 Postype m_posDataBegin; // Read: Data begin position.
389 RposType m_rposEndofHdrData; // Read: rpos of end of header data.
390 NumType m_nReadEntrycount; // Read: Number of entries.
391
392 NumType m_nNextReadHint; // Read: Hint where to start looking for the next read entry.
393
394 };
395
396
397
398 class SsbWrite
399 : public Ssb
400 {
401
402 public:
403
404 SsbWrite(std::ostream& oStrm);
405
406 // Write header
407 void BeginWrite(const ID &id, const uint64& nVersion);
408
409 // Write item using default write implementation.
410 template <class T>
WriteItem(const T & obj,const ID & id)411 void WriteItem(const T& obj, const ID &id) {WriteItem(obj, id, &srlztn::WriteItem<T>);}
412
413 // Write item using given function.
414 template <class T, class FuncObj>
415 void WriteItem(const T& obj, const ID &id, FuncObj);
416
417 // Writes mapping.
418 void FinishWrite();
419
420 private:
421
422 // Called after writing an item.
423 void OnWroteItem(const ID &id, const Postype& posBeforeWrite);
424
425 void AddWriteNote(const SsbStatus s);
426 void AddWriteNote(const ID &id,
427 const NumType nEntryNum,
428 const DataSize nBytecount,
429 const RposType rposStart);
430
431 // Writes mapping item to mapstream.
432 void WriteMapItem(const ID &id,
433 const RposType& rposDataStart,
434 const DataSize& nDatasize,
435 const char* pszDesc);
436
ResetWritestatus()437 void ResetWritestatus() {m_Status = SNT_NONE;}
438
439 void IncrementWriteCounter();
440
441 private:
442
443 std::ostream& oStrm;
444
445 Postype m_posEntrycount; // Write: Pos of entrycount field.
446 Postype m_posMapPosField; // Write: Pos of map position field.
447 std::string m_MapStreamString; // Write: Map stream string.
448
449 };
450
451
452 template <class T, class FuncObj>
WriteItem(const T & obj,const ID & id,FuncObj Func)453 void SsbWrite::WriteItem(const T& obj, const ID &id, FuncObj Func)
454 {
455 const Postype pos = oStrm.tellp();
456 Func(oStrm, obj);
457 OnWroteItem(id, pos);
458 }
459
460 template <class T, class FuncObj>
ReadItem(T & obj,const ID & id,FuncObj Func)461 SsbRead::ReadRv SsbRead::ReadItem(T& obj, const ID &id, FuncObj Func)
462 {
463 const ReadEntry* pE = Find(id);
464 const Postype pos = iStrm.tellg();
465 if (pE != nullptr || GetFlag(RwfRMapHasId) == false)
466 Func(iStrm, obj, (pE) ? (pE->nSize) : invalidDatasize);
467 return OnReadEntry(pE, id, pos);
468 }
469
470
471 template <class T, class FuncObj>
ReadIterItem(const ReadIterator & iter,T & obj,FuncObj func)472 SsbRead::ReadRv SsbRead::ReadIterItem(const ReadIterator& iter, T& obj, FuncObj func)
473 {
474 iStrm.clear();
475 if (iter->rposStart != 0)
476 iStrm.seekg(m_posStart + Postype(iter->rposStart));
477 const Postype pos = iStrm.tellg();
478 func(iStrm, obj, iter->nSize);
479 return OnReadEntry(&(*iter), ID(&m_Idarray[iter->nIdpos], iter->nIdLength), pos);
480 }
481
482
CompareId(const ReadIterator & iter,const ID & id)483 inline SsbRead::IdMatchStatus SsbRead::CompareId(const ReadIterator& iter, const ID &id)
484 {
485 if(iter->nIdpos >= m_Idarray.size()) return IdMismatch;
486 return (id == ID(&m_Idarray[iter->nIdpos], iter->nIdLength)) ? IdMatch : IdMismatch;
487 }
488
489
GetReadBegin()490 inline SsbRead::ReadIterator SsbRead::GetReadBegin()
491 {
492 MPT_ASSERT(GetFlag(RwfRMapHasId) && (GetFlag(RwfRMapHasStartpos) || GetFlag(RwfRMapHasSize) || m_nFixedEntrySize > 0));
493 if (GetFlag(RwfRMapCached) == false)
494 CacheMap();
495 return mapData.begin();
496 }
497
498
GetReadEnd()499 inline SsbRead::ReadIterator SsbRead::GetReadEnd()
500 {
501 if (GetFlag(RwfRMapCached) == false)
502 CacheMap();
503 return mapData.end();
504 }
505
506
507 template <class T>
508 struct VectorWriter
509 {
VectorWriterVectorWriter510 VectorWriter(size_t nCount) : m_nCount(nCount) {}
operatorVectorWriter511 void operator()(std::ostream &oStrm, const std::vector<T> &vec)
512 {
513 for(size_t i = 0; i < m_nCount; i++)
514 {
515 Binarywrite(oStrm, vec[i]);
516 }
517 }
518 size_t m_nCount;
519 };
520
521 template <class T>
522 struct VectorReader
523 {
VectorReaderVectorReader524 VectorReader(size_t nCount) : m_nCount(nCount) {}
operatorVectorReader525 void operator()(std::istream& iStrm, std::vector<T> &vec, const size_t)
526 {
527 vec.resize(m_nCount);
528 for(std::size_t i = 0; i < m_nCount; ++i)
529 {
530 Binaryread(iStrm, vec[i]);
531 }
532 }
533 size_t m_nCount;
534 };
535
536 template <class T>
537 struct ArrayReader
538 {
ArrayReaderArrayReader539 ArrayReader(size_t nCount) : m_nCount(nCount) {}
operatorArrayReader540 void operator()(std::istream& iStrm, T* pData, const size_t)
541 {
542 for(std::size_t i=0; i<m_nCount; ++i)
543 {
544 Binaryread(iStrm, pData[i]);
545 }
546 }
547 size_t m_nCount;
548 };
549
550
551
552 } //namespace srlztn.
553
554
555 OPENMPT_NAMESPACE_END
556