1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2019 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5
6 #ifndef BITCOIN_SERIALIZE_H
7 #define BITCOIN_SERIALIZE_H
8
9 #include <compat/endian.h>
10
11 #include <algorithm>
12 #include <cstring>
13 #include <ios>
14 #include <limits>
15 #include <map>
16 #include <memory>
17 #include <set>
18 #include <stdint.h>
19 #include <string>
20 #include <string.h>
21 #include <utility>
22 #include <vector>
23
24 #include <prevector.h>
25 #include <span.h>
26
27 static const unsigned int MAX_SIZE = 0x10000000; // Qtum: Increase max serialized size to 256mb
28
29 /** Maximum amount of memory (in bytes) to allocate at once when deserializing vectors. */
30 static const unsigned int MAX_VECTOR_ALLOCATE = 5000000;
31
32 /**
33 * Dummy data type to identify deserializing constructors.
34 *
35 * By convention, a constructor of a type T with signature
36 *
37 * template <typename Stream> T::T(deserialize_type, Stream& s)
38 *
39 * is a deserializing constructor, which builds the type by
40 * deserializing it from s. If T contains const fields, this
41 * is likely the only way to do so.
42 */
43 struct deserialize_type {};
44 constexpr deserialize_type deserialize {};
45
46 /**
47 * Used to bypass the rule against non-const reference to temporary
48 * where it makes sense with wrappers.
49 */
50 template<typename T>
REF(const T & val)51 inline T& REF(const T& val)
52 {
53 return const_cast<T&>(val);
54 }
55
56 /**
57 * Used to acquire a non-const pointer "this" to generate bodies
58 * of const serialization operations from a template
59 */
60 template<typename T>
NCONST_PTR(const T * val)61 inline T* NCONST_PTR(const T* val)
62 {
63 return const_cast<T*>(val);
64 }
65
66 //! Safely convert odd char pointer types to standard ones.
CharCast(char * c)67 inline char* CharCast(char* c) { return c; }
CharCast(unsigned char * c)68 inline char* CharCast(unsigned char* c) { return (char*)c; }
CharCast(const char * c)69 inline const char* CharCast(const char* c) { return c; }
CharCast(const unsigned char * c)70 inline const char* CharCast(const unsigned char* c) { return (const char*)c; }
71
72 /*
73 * Lowest-level serialization and conversion.
74 * @note Sizes of these types are verified in the tests
75 */
ser_writedata8(Stream & s,uint8_t obj)76 template<typename Stream> inline void ser_writedata8(Stream &s, uint8_t obj)
77 {
78 s.write((char*)&obj, 1);
79 }
ser_writedata16(Stream & s,uint16_t obj)80 template<typename Stream> inline void ser_writedata16(Stream &s, uint16_t obj)
81 {
82 obj = htole16(obj);
83 s.write((char*)&obj, 2);
84 }
ser_writedata16be(Stream & s,uint16_t obj)85 template<typename Stream> inline void ser_writedata16be(Stream &s, uint16_t obj)
86 {
87 obj = htobe16(obj);
88 s.write((char*)&obj, 2);
89 }
ser_writedata32(Stream & s,uint32_t obj)90 template<typename Stream> inline void ser_writedata32(Stream &s, uint32_t obj)
91 {
92 obj = htole32(obj);
93 s.write((char*)&obj, 4);
94 }
ser_writedata32be(Stream & s,uint32_t obj)95 template<typename Stream> inline void ser_writedata32be(Stream &s, uint32_t obj)
96 {
97 obj = htobe32(obj);
98 s.write((char*)&obj, 4);
99 }
ser_writedata64(Stream & s,uint64_t obj)100 template<typename Stream> inline void ser_writedata64(Stream &s, uint64_t obj)
101 {
102 obj = htole64(obj);
103 s.write((char*)&obj, 8);
104 }
ser_readdata8(Stream & s)105 template<typename Stream> inline uint8_t ser_readdata8(Stream &s)
106 {
107 uint8_t obj;
108 s.read((char*)&obj, 1);
109 return obj;
110 }
ser_readdata16(Stream & s)111 template<typename Stream> inline uint16_t ser_readdata16(Stream &s)
112 {
113 uint16_t obj;
114 s.read((char*)&obj, 2);
115 return le16toh(obj);
116 }
ser_readdata16be(Stream & s)117 template<typename Stream> inline uint16_t ser_readdata16be(Stream &s)
118 {
119 uint16_t obj;
120 s.read((char*)&obj, 2);
121 return be16toh(obj);
122 }
ser_readdata32(Stream & s)123 template<typename Stream> inline uint32_t ser_readdata32(Stream &s)
124 {
125 uint32_t obj;
126 s.read((char*)&obj, 4);
127 return le32toh(obj);
128 }
ser_readdata32be(Stream & s)129 template<typename Stream> inline uint32_t ser_readdata32be(Stream &s)
130 {
131 uint32_t obj;
132 s.read((char*)&obj, 4);
133 return be32toh(obj);
134 }
ser_readdata64(Stream & s)135 template<typename Stream> inline uint64_t ser_readdata64(Stream &s)
136 {
137 uint64_t obj;
138 s.read((char*)&obj, 8);
139 return le64toh(obj);
140 }
ser_double_to_uint64(double x)141 inline uint64_t ser_double_to_uint64(double x)
142 {
143 uint64_t tmp;
144 std::memcpy(&tmp, &x, sizeof(x));
145 static_assert(sizeof(tmp) == sizeof(x), "double and uint64_t assumed to have the same size");
146 return tmp;
147 }
ser_float_to_uint32(float x)148 inline uint32_t ser_float_to_uint32(float x)
149 {
150 uint32_t tmp;
151 std::memcpy(&tmp, &x, sizeof(x));
152 static_assert(sizeof(tmp) == sizeof(x), "float and uint32_t assumed to have the same size");
153 return tmp;
154 }
ser_uint64_to_double(uint64_t y)155 inline double ser_uint64_to_double(uint64_t y)
156 {
157 double tmp;
158 std::memcpy(&tmp, &y, sizeof(y));
159 static_assert(sizeof(tmp) == sizeof(y), "double and uint64_t assumed to have the same size");
160 return tmp;
161 }
ser_uint32_to_float(uint32_t y)162 inline float ser_uint32_to_float(uint32_t y)
163 {
164 float tmp;
165 std::memcpy(&tmp, &y, sizeof(y));
166 static_assert(sizeof(tmp) == sizeof(y), "float and uint32_t assumed to have the same size");
167 return tmp;
168 }
169
170
171 /////////////////////////////////////////////////////////////////
172 //
173 // Templates for serializing to anything that looks like a stream,
174 // i.e. anything that supports .read(char*, size_t) and .write(char*, size_t)
175 //
176
177 class CSizeComputer;
178
179 enum
180 {
181 // primary actions
182 SER_NETWORK = (1 << 0),
183 SER_DISK = (1 << 1),
184 SER_GETHASH = (1 << 2),
185 };
186
187 //! Convert the reference base type to X, without changing constness or reference type.
ReadWriteAsHelper(X & x)188 template<typename X> X& ReadWriteAsHelper(X& x) { return x; }
ReadWriteAsHelper(const X & x)189 template<typename X> const X& ReadWriteAsHelper(const X& x) { return x; }
190
191 #define READWRITE(...) (::SerReadWriteMany(s, ser_action, __VA_ARGS__))
192 #define READWRITEAS(type, obj) (::SerReadWriteMany(s, ser_action, ReadWriteAsHelper<type>(obj)))
193
194 /**
195 * Implement three methods for serializable objects. These are actually wrappers over
196 * "SerializationOp" template, which implements the body of each class' serialization
197 * code. Adding "ADD_SERIALIZE_METHODS" in the body of the class causes these wrappers to be
198 * added as members.
199 */
200 #define ADD_SERIALIZE_METHODS \
201 template<typename Stream> \
202 void Serialize(Stream& s) const { \
203 NCONST_PTR(this)->SerializationOp(s, CSerActionSerialize()); \
204 } \
205 template<typename Stream> \
206 void Unserialize(Stream& s) { \
207 SerializationOp(s, CSerActionUnserialize()); \
208 }
209
210 /**
211 * Implement the Ser and Unser methods needed for implementing a formatter (see Using below).
212 *
213 * Both Ser and Unser are delegated to a single static method SerializationOps, which is polymorphic
214 * in the serialized/deserialized type (allowing it to be const when serializing, and non-const when
215 * deserializing).
216 *
217 * Example use:
218 * struct FooFormatter {
219 * FORMATTER_METHODS(Class, obj) { READWRITE(obj.val1, VARINT(obj.val2)); }
220 * }
221 * would define a class FooFormatter that defines a serialization of Class objects consisting
222 * of serializing its val1 member using the default serialization, and its val2 member using
223 * VARINT serialization. That FooFormatter can then be used in statements like
224 * READWRITE(Using<FooFormatter>(obj.bla)).
225 */
226 #define FORMATTER_METHODS(cls, obj) \
227 template<typename Stream> \
228 static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, CSerActionSerialize()); } \
229 template<typename Stream> \
230 static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, CSerActionUnserialize()); } \
231 template<typename Stream, typename Type, typename Operation> \
232 static inline void SerializationOps(Type& obj, Stream& s, Operation ser_action) \
233
234 /**
235 * Implement the Serialize and Unserialize methods by delegating to a single templated
236 * static method that takes the to-be-(de)serialized object as a parameter. This approach
237 * has the advantage that the constness of the object becomes a template parameter, and
238 * thus allows a single implementation that sees the object as const for serializing
239 * and non-const for deserializing, without casts.
240 */
241 #define SERIALIZE_METHODS(cls, obj) \
242 template<typename Stream> \
243 void Serialize(Stream& s) const \
244 { \
245 static_assert(std::is_same<const cls&, decltype(*this)>::value, "Serialize type mismatch"); \
246 Ser(s, *this); \
247 } \
248 template<typename Stream> \
249 void Unserialize(Stream& s) \
250 { \
251 static_assert(std::is_same<cls&, decltype(*this)>::value, "Unserialize type mismatch"); \
252 Unser(s, *this); \
253 } \
254 FORMATTER_METHODS(cls, obj)
255
256 #ifndef CHAR_EQUALS_INT8
Serialize(Stream & s,char a)257 template<typename Stream> inline void Serialize(Stream& s, char a ) { ser_writedata8(s, a); } // TODO Get rid of bare char
258 #endif
Serialize(Stream & s,int8_t a)259 template<typename Stream> inline void Serialize(Stream& s, int8_t a ) { ser_writedata8(s, a); }
Serialize(Stream & s,uint8_t a)260 template<typename Stream> inline void Serialize(Stream& s, uint8_t a ) { ser_writedata8(s, a); }
Serialize(Stream & s,int16_t a)261 template<typename Stream> inline void Serialize(Stream& s, int16_t a ) { ser_writedata16(s, a); }
Serialize(Stream & s,uint16_t a)262 template<typename Stream> inline void Serialize(Stream& s, uint16_t a) { ser_writedata16(s, a); }
Serialize(Stream & s,int32_t a)263 template<typename Stream> inline void Serialize(Stream& s, int32_t a ) { ser_writedata32(s, a); }
Serialize(Stream & s,uint32_t a)264 template<typename Stream> inline void Serialize(Stream& s, uint32_t a) { ser_writedata32(s, a); }
Serialize(Stream & s,int64_t a)265 template<typename Stream> inline void Serialize(Stream& s, int64_t a ) { ser_writedata64(s, a); }
Serialize(Stream & s,uint64_t a)266 template<typename Stream> inline void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); }
Serialize(Stream & s,float a)267 template<typename Stream> inline void Serialize(Stream& s, float a ) { ser_writedata32(s, ser_float_to_uint32(a)); }
Serialize(Stream & s,double a)268 template<typename Stream> inline void Serialize(Stream& s, double a ) { ser_writedata64(s, ser_double_to_uint64(a)); }
Serialize(Stream & s,const char (& a)[N])269 template<typename Stream, int N> inline void Serialize(Stream& s, const char (&a)[N]) { s.write(a, N); }
Serialize(Stream & s,const unsigned char (& a)[N])270 template<typename Stream, int N> inline void Serialize(Stream& s, const unsigned char (&a)[N]) { s.write(CharCast(a), N); }
Serialize(Stream & s,const Span<const unsigned char> & span)271 template<typename Stream> inline void Serialize(Stream& s, const Span<const unsigned char>& span) { s.write(CharCast(span.data()), span.size()); }
Serialize(Stream & s,const Span<unsigned char> & span)272 template<typename Stream> inline void Serialize(Stream& s, const Span<unsigned char>& span) { s.write(CharCast(span.data()), span.size()); }
273
274 #ifndef CHAR_EQUALS_INT8
Unserialize(Stream & s,char & a)275 template<typename Stream> inline void Unserialize(Stream& s, char& a ) { a = ser_readdata8(s); } // TODO Get rid of bare char
276 #endif
Unserialize(Stream & s,int8_t & a)277 template<typename Stream> inline void Unserialize(Stream& s, int8_t& a ) { a = ser_readdata8(s); }
Unserialize(Stream & s,uint8_t & a)278 template<typename Stream> inline void Unserialize(Stream& s, uint8_t& a ) { a = ser_readdata8(s); }
Unserialize(Stream & s,int16_t & a)279 template<typename Stream> inline void Unserialize(Stream& s, int16_t& a ) { a = ser_readdata16(s); }
Unserialize(Stream & s,uint16_t & a)280 template<typename Stream> inline void Unserialize(Stream& s, uint16_t& a) { a = ser_readdata16(s); }
Unserialize(Stream & s,int32_t & a)281 template<typename Stream> inline void Unserialize(Stream& s, int32_t& a ) { a = ser_readdata32(s); }
Unserialize(Stream & s,uint32_t & a)282 template<typename Stream> inline void Unserialize(Stream& s, uint32_t& a) { a = ser_readdata32(s); }
Unserialize(Stream & s,int64_t & a)283 template<typename Stream> inline void Unserialize(Stream& s, int64_t& a ) { a = ser_readdata64(s); }
Unserialize(Stream & s,uint64_t & a)284 template<typename Stream> inline void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); }
Unserialize(Stream & s,float & a)285 template<typename Stream> inline void Unserialize(Stream& s, float& a ) { a = ser_uint32_to_float(ser_readdata32(s)); }
Unserialize(Stream & s,double & a)286 template<typename Stream> inline void Unserialize(Stream& s, double& a ) { a = ser_uint64_to_double(ser_readdata64(s)); }
Unserialize(Stream & s,char (& a)[N])287 template<typename Stream, int N> inline void Unserialize(Stream& s, char (&a)[N]) { s.read(a, N); }
Unserialize(Stream & s,unsigned char (& a)[N])288 template<typename Stream, int N> inline void Unserialize(Stream& s, unsigned char (&a)[N]) { s.read(CharCast(a), N); }
Unserialize(Stream & s,Span<unsigned char> & span)289 template<typename Stream> inline void Unserialize(Stream& s, Span<unsigned char>& span) { s.read(CharCast(span.data()), span.size()); }
290
Serialize(Stream & s,bool a)291 template<typename Stream> inline void Serialize(Stream& s, bool a) { char f=a; ser_writedata8(s, f); }
Unserialize(Stream & s,bool & a)292 template<typename Stream> inline void Unserialize(Stream& s, bool& a) { char f=ser_readdata8(s); a=f; }
293
294
295
296
297
298
299 /**
300 * Compact Size
301 * size < 253 -- 1 byte
302 * size <= USHRT_MAX -- 3 bytes (253 + 2 bytes)
303 * size <= UINT_MAX -- 5 bytes (254 + 4 bytes)
304 * size > UINT_MAX -- 9 bytes (255 + 8 bytes)
305 */
GetSizeOfCompactSize(uint64_t nSize)306 inline unsigned int GetSizeOfCompactSize(uint64_t nSize)
307 {
308 if (nSize < 253) return sizeof(unsigned char);
309 else if (nSize <= std::numeric_limits<unsigned short>::max()) return sizeof(unsigned char) + sizeof(unsigned short);
310 else if (nSize <= std::numeric_limits<unsigned int>::max()) return sizeof(unsigned char) + sizeof(unsigned int);
311 else return sizeof(unsigned char) + sizeof(uint64_t);
312 }
313
314 inline void WriteCompactSize(CSizeComputer& os, uint64_t nSize);
315
316 template<typename Stream>
WriteCompactSize(Stream & os,uint64_t nSize)317 void WriteCompactSize(Stream& os, uint64_t nSize)
318 {
319 if (nSize < 253)
320 {
321 ser_writedata8(os, nSize);
322 }
323 else if (nSize <= std::numeric_limits<unsigned short>::max())
324 {
325 ser_writedata8(os, 253);
326 ser_writedata16(os, nSize);
327 }
328 else if (nSize <= std::numeric_limits<unsigned int>::max())
329 {
330 ser_writedata8(os, 254);
331 ser_writedata32(os, nSize);
332 }
333 else
334 {
335 ser_writedata8(os, 255);
336 ser_writedata64(os, nSize);
337 }
338 return;
339 }
340
341 template<typename Stream>
ReadCompactSize(Stream & is)342 uint64_t ReadCompactSize(Stream& is)
343 {
344 uint8_t chSize = ser_readdata8(is);
345 uint64_t nSizeRet = 0;
346 if (chSize < 253)
347 {
348 nSizeRet = chSize;
349 }
350 else if (chSize == 253)
351 {
352 nSizeRet = ser_readdata16(is);
353 if (nSizeRet < 253)
354 throw std::ios_base::failure("non-canonical ReadCompactSize()");
355 }
356 else if (chSize == 254)
357 {
358 nSizeRet = ser_readdata32(is);
359 if (nSizeRet < 0x10000u)
360 throw std::ios_base::failure("non-canonical ReadCompactSize()");
361 }
362 else
363 {
364 nSizeRet = ser_readdata64(is);
365 if (nSizeRet < 0x100000000ULL)
366 throw std::ios_base::failure("non-canonical ReadCompactSize()");
367 }
368 if (nSizeRet > (uint64_t)MAX_SIZE)
369 throw std::ios_base::failure("ReadCompactSize(): size too large");
370 return nSizeRet;
371 }
372
373 /**
374 * Variable-length integers: bytes are a MSB base-128 encoding of the number.
375 * The high bit in each byte signifies whether another digit follows. To make
376 * sure the encoding is one-to-one, one is subtracted from all but the last digit.
377 * Thus, the byte sequence a[] with length len, where all but the last byte
378 * has bit 128 set, encodes the number:
379 *
380 * (a[len-1] & 0x7F) + sum(i=1..len-1, 128^i*((a[len-i-1] & 0x7F)+1))
381 *
382 * Properties:
383 * * Very small (0-127: 1 byte, 128-16511: 2 bytes, 16512-2113663: 3 bytes)
384 * * Every integer has exactly one encoding
385 * * Encoding does not depend on size of original integer type
386 * * No redundancy: every (infinite) byte sequence corresponds to a list
387 * of encoded integers.
388 *
389 * 0: [0x00] 256: [0x81 0x00]
390 * 1: [0x01] 16383: [0xFE 0x7F]
391 * 127: [0x7F] 16384: [0xFF 0x00]
392 * 128: [0x80 0x00] 16511: [0xFF 0x7F]
393 * 255: [0x80 0x7F] 65535: [0x82 0xFE 0x7F]
394 * 2^32: [0x8E 0xFE 0xFE 0xFF 0x00]
395 */
396
397 /**
398 * Mode for encoding VarInts.
399 *
400 * Currently there is no support for signed encodings. The default mode will not
401 * compile with signed values, and the legacy "nonnegative signed" mode will
402 * accept signed values, but improperly encode and decode them if they are
403 * negative. In the future, the DEFAULT mode could be extended to support
404 * negative numbers in a backwards compatible way, and additional modes could be
405 * added to support different varint formats (e.g. zigzag encoding).
406 */
407 enum class VarIntMode { DEFAULT, NONNEGATIVE_SIGNED };
408
409 template <VarIntMode Mode, typename I>
410 struct CheckVarIntMode {
CheckVarIntModeCheckVarIntMode411 constexpr CheckVarIntMode()
412 {
413 static_assert(Mode != VarIntMode::DEFAULT || std::is_unsigned<I>::value, "Unsigned type required with mode DEFAULT.");
414 static_assert(Mode != VarIntMode::NONNEGATIVE_SIGNED || std::is_signed<I>::value, "Signed type required with mode NONNEGATIVE_SIGNED.");
415 }
416 };
417
418 template<VarIntMode Mode, typename I>
GetSizeOfVarInt(I n)419 inline unsigned int GetSizeOfVarInt(I n)
420 {
421 CheckVarIntMode<Mode, I>();
422 int nRet = 0;
423 while(true) {
424 nRet++;
425 if (n <= 0x7F)
426 break;
427 n = (n >> 7) - 1;
428 }
429 return nRet;
430 }
431
432 template<typename I>
433 inline void WriteVarInt(CSizeComputer& os, I n);
434
435 template<typename Stream, VarIntMode Mode, typename I>
WriteVarInt(Stream & os,I n)436 void WriteVarInt(Stream& os, I n)
437 {
438 CheckVarIntMode<Mode, I>();
439 unsigned char tmp[(sizeof(n)*8+6)/7];
440 int len=0;
441 while(true) {
442 tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00);
443 if (n <= 0x7F)
444 break;
445 n = (n >> 7) - 1;
446 len++;
447 }
448 do {
449 ser_writedata8(os, tmp[len]);
450 } while(len--);
451 }
452
453 template<typename Stream, VarIntMode Mode, typename I>
ReadVarInt(Stream & is)454 I ReadVarInt(Stream& is)
455 {
456 CheckVarIntMode<Mode, I>();
457 I n = 0;
458 while(true) {
459 unsigned char chData = ser_readdata8(is);
460 if (n > (std::numeric_limits<I>::max() >> 7)) {
461 throw std::ios_base::failure("ReadVarInt(): size too large");
462 }
463 n = (n << 7) | (chData & 0x7F);
464 if (chData & 0x80) {
465 if (n == std::numeric_limits<I>::max()) {
466 throw std::ios_base::failure("ReadVarInt(): size too large");
467 }
468 n++;
469 } else {
470 return n;
471 }
472 }
473 }
474
475 /** Simple wrapper class to serialize objects using a formatter; used by Using(). */
476 template<typename Formatter, typename T>
477 class Wrapper
478 {
479 static_assert(std::is_lvalue_reference<T>::value, "Wrapper needs an lvalue reference type T");
480 protected:
481 T m_object;
482 public:
Wrapper(T obj)483 explicit Wrapper(T obj) : m_object(obj) {}
Serialize(Stream & s)484 template<typename Stream> void Serialize(Stream &s) const { Formatter().Ser(s, m_object); }
Unserialize(Stream & s)485 template<typename Stream> void Unserialize(Stream &s) { Formatter().Unser(s, m_object); }
486 };
487
488 /** Cause serialization/deserialization of an object to be done using a specified formatter class.
489 *
490 * To use this, you need a class Formatter that has public functions Ser(stream, const object&) for
491 * serialization, and Unser(stream, object&) for deserialization. Serialization routines (inside
492 * READWRITE, or directly with << and >> operators), can then use Using<Formatter>(object).
493 *
494 * This works by constructing a Wrapper<Formatter, T>-wrapped version of object, where T is
495 * const during serialization, and non-const during deserialization, which maintains const
496 * correctness.
497 */
498 template<typename Formatter, typename T>
Using(T && t)499 static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T&>(t); }
500
501 #define VARINT_MODE(obj, mode) Using<VarIntFormatter<mode>>(obj)
502 #define VARINT(obj) Using<VarIntFormatter<VarIntMode::DEFAULT>>(obj)
503 #define COMPACTSIZE(obj) Using<CompactSizeFormatter>(obj)
504 #define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj))
505
506 /** Serialization wrapper class for integers in VarInt format. */
507 template<VarIntMode Mode>
508 struct VarIntFormatter
509 {
SerVarIntFormatter510 template<typename Stream, typename I> void Ser(Stream &s, I v)
511 {
512 WriteVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s, v);
513 }
514
UnserVarIntFormatter515 template<typename Stream, typename I> void Unser(Stream& s, I& v)
516 {
517 v = ReadVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s);
518 }
519 };
520
521 template<int Bytes>
522 struct CustomUintFormatter
523 {
524 static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range");
525 static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes));
526
SerCustomUintFormatter527 template <typename Stream, typename I> void Ser(Stream& s, I v)
528 {
529 if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range");
530 uint64_t raw = htole64(v);
531 s.write((const char*)&raw, Bytes);
532 }
533
UnserCustomUintFormatter534 template <typename Stream, typename I> void Unser(Stream& s, I& v)
535 {
536 static_assert(std::numeric_limits<I>::max() >= MAX && std::numeric_limits<I>::min() <= 0, "CustomUintFormatter type too small");
537 uint64_t raw = 0;
538 s.read((char*)&raw, Bytes);
539 v = le64toh(raw);
540 }
541 };
542
543 /** Serialization wrapper class for big-endian integers.
544 *
545 * Use this wrapper around integer types that are stored in memory in native
546 * byte order, but serialized in big endian notation. This is only intended
547 * to implement serializers that are compatible with existing formats, and
548 * its use is not recommended for new data structures.
549 *
550 * Only 16-bit types are supported for now.
551 */
552 template<typename I>
553 class BigEndian
554 {
555 protected:
556 I& m_val;
557 public:
BigEndian(I & val)558 explicit BigEndian(I& val) : m_val(val)
559 {
560 static_assert(std::is_unsigned<I>::value, "BigEndian type must be unsigned integer");
561 static_assert(sizeof(I) == 2 && std::numeric_limits<I>::min() == 0 && std::numeric_limits<I>::max() == std::numeric_limits<uint16_t>::max(), "Unsupported BigEndian size");
562 }
563
564 template<typename Stream>
Serialize(Stream & s)565 void Serialize(Stream& s) const
566 {
567 ser_writedata16be(s, m_val);
568 }
569
570 template<typename Stream>
Unserialize(Stream & s)571 void Unserialize(Stream& s)
572 {
573 m_val = ser_readdata16be(s);
574 }
575 };
576
577 /** Formatter for integers in CompactSize format. */
578 struct CompactSizeFormatter
579 {
580 template<typename Stream, typename I>
UnserCompactSizeFormatter581 void Unser(Stream& s, I& v)
582 {
583 uint64_t n = ReadCompactSize<Stream>(s);
584 if (n < std::numeric_limits<I>::min() || n > std::numeric_limits<I>::max()) {
585 throw std::ios_base::failure("CompactSize exceeds limit of type");
586 }
587 v = n;
588 }
589
590 template<typename Stream, typename I>
SerCompactSizeFormatter591 void Ser(Stream& s, I v)
592 {
593 static_assert(std::is_unsigned<I>::value, "CompactSize only supported for unsigned integers");
594 static_assert(std::numeric_limits<I>::max() <= std::numeric_limits<uint64_t>::max(), "CompactSize only supports 64-bit integers and below");
595
596 WriteCompactSize<Stream>(s, v);
597 }
598 };
599
600 template<size_t Limit>
601 class LimitedString
602 {
603 protected:
604 std::string& string;
605 public:
LimitedString(std::string & _string)606 explicit LimitedString(std::string& _string) : string(_string) {}
607
608 template<typename Stream>
Unserialize(Stream & s)609 void Unserialize(Stream& s)
610 {
611 size_t size = ReadCompactSize(s);
612 if (size > Limit) {
613 throw std::ios_base::failure("String length limit exceeded");
614 }
615 string.resize(size);
616 if (size != 0)
617 s.read((char*)string.data(), size);
618 }
619
620 template<typename Stream>
Serialize(Stream & s)621 void Serialize(Stream& s) const
622 {
623 WriteCompactSize(s, string.size());
624 if (!string.empty())
625 s.write((char*)string.data(), string.size());
626 }
627 };
628
629 template<typename I>
WrapBigEndian(I & n)630 BigEndian<I> WrapBigEndian(I& n) { return BigEndian<I>(n); }
631
632 /** Formatter to serialize/deserialize vector elements using another formatter
633 *
634 * Example:
635 * struct X {
636 * std::vector<uint64_t> v;
637 * SERIALIZE_METHODS(X, obj) { READWRITE(Using<VectorFormatter<VarInt>>(obj.v)); }
638 * };
639 * will define a struct that contains a vector of uint64_t, which is serialized
640 * as a vector of VarInt-encoded integers.
641 *
642 * V is not required to be an std::vector type. It works for any class that
643 * exposes a value_type, size, reserve, emplace_back, back, and const iterators.
644 */
645 template<class Formatter>
646 struct VectorFormatter
647 {
648 template<typename Stream, typename V>
SerVectorFormatter649 void Ser(Stream& s, const V& v)
650 {
651 Formatter formatter;
652 WriteCompactSize(s, v.size());
653 for (const typename V::value_type& elem : v) {
654 formatter.Ser(s, elem);
655 }
656 }
657
658 template<typename Stream, typename V>
UnserVectorFormatter659 void Unser(Stream& s, V& v)
660 {
661 Formatter formatter;
662 v.clear();
663 size_t size = ReadCompactSize(s);
664 size_t allocated = 0;
665 while (allocated < size) {
666 // For DoS prevention, do not blindly allocate as much as the stream claims to contain.
667 // Instead, allocate in 5MiB batches, so that an attacker actually needs to provide
668 // X MiB of data to make us allocate X+5 Mib.
669 static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE, "Vector element size too large");
670 allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type));
671 v.reserve(allocated);
672 while (v.size() < allocated) {
673 v.emplace_back();
674 formatter.Unser(s, v.back());
675 }
676 }
677 };
678 };
679
680 /**
681 * Forward declarations
682 */
683
684 /**
685 * string
686 */
687 template<typename Stream, typename C> void Serialize(Stream& os, const std::basic_string<C>& str);
688 template<typename Stream, typename C> void Unserialize(Stream& is, std::basic_string<C>& str);
689
690 /**
691 * prevector
692 * prevectors of unsigned char are a special case and are intended to be serialized as a single opaque blob.
693 */
694 template<typename Stream, unsigned int N, typename T> void Serialize_impl(Stream& os, const prevector<N, T>& v, const unsigned char&);
695 template<typename Stream, unsigned int N, typename T, typename V> void Serialize_impl(Stream& os, const prevector<N, T>& v, const V&);
696 template<typename Stream, unsigned int N, typename T> inline void Serialize(Stream& os, const prevector<N, T>& v);
697 template<typename Stream, unsigned int N, typename T> void Unserialize_impl(Stream& is, prevector<N, T>& v, const unsigned char&);
698 template<typename Stream, unsigned int N, typename T, typename V> void Unserialize_impl(Stream& is, prevector<N, T>& v, const V&);
699 template<typename Stream, unsigned int N, typename T> inline void Unserialize(Stream& is, prevector<N, T>& v);
700
701 /**
702 * vector
703 * vectors of unsigned char are a special case and are intended to be serialized as a single opaque blob.
704 */
705 template<typename Stream, typename T, typename A> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const unsigned char&);
706 template<typename Stream, typename T, typename A> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const bool&);
707 template<typename Stream, typename T, typename A, typename V> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&);
708 template<typename Stream, typename T, typename A> inline void Serialize(Stream& os, const std::vector<T, A>& v);
709 template<typename Stream, typename T, typename A> void Unserialize_impl(Stream& is, std::vector<T, A>& v, const unsigned char&);
710 template<typename Stream, typename T, typename A, typename V> void Unserialize_impl(Stream& is, std::vector<T, A>& v, const V&);
711 template<typename Stream, typename T, typename A> inline void Unserialize(Stream& is, std::vector<T, A>& v);
712
713 /**
714 * pair
715 */
716 template<typename Stream, typename K, typename T> void Serialize(Stream& os, const std::pair<K, T>& item);
717 template<typename Stream, typename K, typename T> void Unserialize(Stream& is, std::pair<K, T>& item);
718
719 /**
720 * map
721 */
722 template<typename Stream, typename K, typename T, typename Pred, typename A> void Serialize(Stream& os, const std::map<K, T, Pred, A>& m);
723 template<typename Stream, typename K, typename T, typename Pred, typename A> void Unserialize(Stream& is, std::map<K, T, Pred, A>& m);
724
725 /**
726 * set
727 */
728 template<typename Stream, typename K, typename Pred, typename A> void Serialize(Stream& os, const std::set<K, Pred, A>& m);
729 template<typename Stream, typename K, typename Pred, typename A> void Unserialize(Stream& is, std::set<K, Pred, A>& m);
730
731 /**
732 * shared_ptr
733 */
734 template<typename Stream, typename T> void Serialize(Stream& os, const std::shared_ptr<const T>& p);
735 template<typename Stream, typename T> void Unserialize(Stream& os, std::shared_ptr<const T>& p);
736
737 /**
738 * unique_ptr
739 */
740 template<typename Stream, typename T> void Serialize(Stream& os, const std::unique_ptr<const T>& p);
741 template<typename Stream, typename T> void Unserialize(Stream& os, std::unique_ptr<const T>& p);
742
743
744
745 /**
746 * If none of the specialized versions above matched, default to calling member function.
747 */
748 template<typename Stream, typename T>
Serialize(Stream & os,const T & a)749 inline void Serialize(Stream& os, const T& a)
750 {
751 a.Serialize(os);
752 }
753
754 template<typename Stream, typename T>
Unserialize(Stream & is,T && a)755 inline void Unserialize(Stream& is, T&& a)
756 {
757 a.Unserialize(is);
758 }
759
760 /** Default formatter. Serializes objects as themselves.
761 *
762 * The vector/prevector serialization code passes this to VectorFormatter
763 * to enable reusing that logic. It shouldn't be needed elsewhere.
764 */
765 struct DefaultFormatter
766 {
767 template<typename Stream, typename T>
SerDefaultFormatter768 static void Ser(Stream& s, const T& t) { Serialize(s, t); }
769
770 template<typename Stream, typename T>
UnserDefaultFormatter771 static void Unser(Stream& s, T& t) { Unserialize(s, t); }
772 };
773
774
775
776
777
778 /**
779 * string
780 */
781 template<typename Stream, typename C>
Serialize(Stream & os,const std::basic_string<C> & str)782 void Serialize(Stream& os, const std::basic_string<C>& str)
783 {
784 WriteCompactSize(os, str.size());
785 if (!str.empty())
786 os.write((char*)str.data(), str.size() * sizeof(C));
787 }
788
789 template<typename Stream, typename C>
Unserialize(Stream & is,std::basic_string<C> & str)790 void Unserialize(Stream& is, std::basic_string<C>& str)
791 {
792 unsigned int nSize = ReadCompactSize(is);
793 str.resize(nSize);
794 if (nSize != 0)
795 is.read((char*)str.data(), nSize * sizeof(C));
796 }
797
798
799
800 /**
801 * prevector
802 */
803 template<typename Stream, unsigned int N, typename T>
Serialize_impl(Stream & os,const prevector<N,T> & v,const unsigned char &)804 void Serialize_impl(Stream& os, const prevector<N, T>& v, const unsigned char&)
805 {
806 WriteCompactSize(os, v.size());
807 if (!v.empty())
808 os.write((char*)v.data(), v.size() * sizeof(T));
809 }
810
811 template<typename Stream, unsigned int N, typename T, typename V>
Serialize_impl(Stream & os,const prevector<N,T> & v,const V &)812 void Serialize_impl(Stream& os, const prevector<N, T>& v, const V&)
813 {
814 Serialize(os, Using<VectorFormatter<DefaultFormatter>>(v));
815 }
816
817 template<typename Stream, unsigned int N, typename T>
Serialize(Stream & os,const prevector<N,T> & v)818 inline void Serialize(Stream& os, const prevector<N, T>& v)
819 {
820 Serialize_impl(os, v, T());
821 }
822
823
824 template<typename Stream, unsigned int N, typename T>
Unserialize_impl(Stream & is,prevector<N,T> & v,const unsigned char &)825 void Unserialize_impl(Stream& is, prevector<N, T>& v, const unsigned char&)
826 {
827 // Limit size per read so bogus size value won't cause out of memory
828 v.clear();
829 unsigned int nSize = ReadCompactSize(is);
830 unsigned int i = 0;
831 while (i < nSize)
832 {
833 unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T)));
834 v.resize_uninitialized(i + blk);
835 is.read((char*)&v[i], blk * sizeof(T));
836 i += blk;
837 }
838 }
839
840 template<typename Stream, unsigned int N, typename T, typename V>
Unserialize_impl(Stream & is,prevector<N,T> & v,const V &)841 void Unserialize_impl(Stream& is, prevector<N, T>& v, const V&)
842 {
843 Unserialize(is, Using<VectorFormatter<DefaultFormatter>>(v));
844 }
845
846 template<typename Stream, unsigned int N, typename T>
Unserialize(Stream & is,prevector<N,T> & v)847 inline void Unserialize(Stream& is, prevector<N, T>& v)
848 {
849 Unserialize_impl(is, v, T());
850 }
851
852
853
854 /**
855 * vector
856 */
857 template<typename Stream, typename T, typename A>
Serialize_impl(Stream & os,const std::vector<T,A> & v,const unsigned char &)858 void Serialize_impl(Stream& os, const std::vector<T, A>& v, const unsigned char&)
859 {
860 WriteCompactSize(os, v.size());
861 if (!v.empty())
862 os.write((char*)v.data(), v.size() * sizeof(T));
863 }
864
865 template<typename Stream, typename T, typename A>
Serialize_impl(Stream & os,const std::vector<T,A> & v,const bool &)866 void Serialize_impl(Stream& os, const std::vector<T, A>& v, const bool&)
867 {
868 // A special case for std::vector<bool>, as dereferencing
869 // std::vector<bool>::const_iterator does not result in a const bool&
870 // due to std::vector's special casing for bool arguments.
871 WriteCompactSize(os, v.size());
872 for (bool elem : v) {
873 ::Serialize(os, elem);
874 }
875 }
876
877 template<typename Stream, typename T, typename A, typename V>
Serialize_impl(Stream & os,const std::vector<T,A> & v,const V &)878 void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&)
879 {
880 Serialize(os, Using<VectorFormatter<DefaultFormatter>>(v));
881 }
882
883 template<typename Stream, typename T, typename A>
Serialize(Stream & os,const std::vector<T,A> & v)884 inline void Serialize(Stream& os, const std::vector<T, A>& v)
885 {
886 Serialize_impl(os, v, T());
887 }
888
889
890 template<typename Stream, typename T, typename A>
Unserialize_impl(Stream & is,std::vector<T,A> & v,const unsigned char &)891 void Unserialize_impl(Stream& is, std::vector<T, A>& v, const unsigned char&)
892 {
893 // Limit size per read so bogus size value won't cause out of memory
894 v.clear();
895 unsigned int nSize = ReadCompactSize(is);
896 unsigned int i = 0;
897 while (i < nSize)
898 {
899 unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T)));
900 v.resize(i + blk);
901 is.read((char*)&v[i], blk * sizeof(T));
902 i += blk;
903 }
904 }
905
906 template<typename Stream, typename T, typename A, typename V>
Unserialize_impl(Stream & is,std::vector<T,A> & v,const V &)907 void Unserialize_impl(Stream& is, std::vector<T, A>& v, const V&)
908 {
909 Unserialize(is, Using<VectorFormatter<DefaultFormatter>>(v));
910 }
911
912 template<typename Stream, typename T, typename A>
Unserialize(Stream & is,std::vector<T,A> & v)913 inline void Unserialize(Stream& is, std::vector<T, A>& v)
914 {
915 Unserialize_impl(is, v, T());
916 }
917
918
919
920 /**
921 * pair
922 */
923 template<typename Stream, typename K, typename T>
Serialize(Stream & os,const std::pair<K,T> & item)924 void Serialize(Stream& os, const std::pair<K, T>& item)
925 {
926 Serialize(os, item.first);
927 Serialize(os, item.second);
928 }
929
930 template<typename Stream, typename K, typename T>
Unserialize(Stream & is,std::pair<K,T> & item)931 void Unserialize(Stream& is, std::pair<K, T>& item)
932 {
933 Unserialize(is, item.first);
934 Unserialize(is, item.second);
935 }
936
937
938
939 /**
940 * map
941 */
942 template<typename Stream, typename K, typename T, typename Pred, typename A>
Serialize(Stream & os,const std::map<K,T,Pred,A> & m)943 void Serialize(Stream& os, const std::map<K, T, Pred, A>& m)
944 {
945 WriteCompactSize(os, m.size());
946 for (const auto& entry : m)
947 Serialize(os, entry);
948 }
949
950 template<typename Stream, typename K, typename T, typename Pred, typename A>
Unserialize(Stream & is,std::map<K,T,Pred,A> & m)951 void Unserialize(Stream& is, std::map<K, T, Pred, A>& m)
952 {
953 m.clear();
954 unsigned int nSize = ReadCompactSize(is);
955 typename std::map<K, T, Pred, A>::iterator mi = m.begin();
956 for (unsigned int i = 0; i < nSize; i++)
957 {
958 std::pair<K, T> item;
959 Unserialize(is, item);
960 mi = m.insert(mi, item);
961 }
962 }
963
964
965
966 /**
967 * set
968 */
969 template<typename Stream, typename K, typename Pred, typename A>
Serialize(Stream & os,const std::set<K,Pred,A> & m)970 void Serialize(Stream& os, const std::set<K, Pred, A>& m)
971 {
972 WriteCompactSize(os, m.size());
973 for (typename std::set<K, Pred, A>::const_iterator it = m.begin(); it != m.end(); ++it)
974 Serialize(os, (*it));
975 }
976
977 template<typename Stream, typename K, typename Pred, typename A>
Unserialize(Stream & is,std::set<K,Pred,A> & m)978 void Unserialize(Stream& is, std::set<K, Pred, A>& m)
979 {
980 m.clear();
981 unsigned int nSize = ReadCompactSize(is);
982 typename std::set<K, Pred, A>::iterator it = m.begin();
983 for (unsigned int i = 0; i < nSize; i++)
984 {
985 K key;
986 Unserialize(is, key);
987 it = m.insert(it, key);
988 }
989 }
990
991
992
993 /**
994 * unique_ptr
995 */
996 template<typename Stream, typename T> void
Serialize(Stream & os,const std::unique_ptr<const T> & p)997 Serialize(Stream& os, const std::unique_ptr<const T>& p)
998 {
999 Serialize(os, *p);
1000 }
1001
1002 template<typename Stream, typename T>
Unserialize(Stream & is,std::unique_ptr<const T> & p)1003 void Unserialize(Stream& is, std::unique_ptr<const T>& p)
1004 {
1005 p.reset(new T(deserialize, is));
1006 }
1007
1008
1009
1010 /**
1011 * shared_ptr
1012 */
1013 template<typename Stream, typename T> void
Serialize(Stream & os,const std::shared_ptr<const T> & p)1014 Serialize(Stream& os, const std::shared_ptr<const T>& p)
1015 {
1016 Serialize(os, *p);
1017 }
1018
1019 template<typename Stream, typename T>
Unserialize(Stream & is,std::shared_ptr<const T> & p)1020 void Unserialize(Stream& is, std::shared_ptr<const T>& p)
1021 {
1022 p = std::make_shared<const T>(deserialize, is);
1023 }
1024
1025
1026
1027 /**
1028 * Support for ADD_SERIALIZE_METHODS and READWRITE macro
1029 */
1030 struct CSerActionSerialize
1031 {
ForReadCSerActionSerialize1032 constexpr bool ForRead() const { return false; }
1033 };
1034 struct CSerActionUnserialize
1035 {
ForReadCSerActionUnserialize1036 constexpr bool ForRead() const { return true; }
1037 };
1038
1039
1040
1041
1042
1043
1044
1045
1046 /* ::GetSerializeSize implementations
1047 *
1048 * Computing the serialized size of objects is done through a special stream
1049 * object of type CSizeComputer, which only records the number of bytes written
1050 * to it.
1051 *
1052 * If your Serialize or SerializationOp method has non-trivial overhead for
1053 * serialization, it may be worthwhile to implement a specialized version for
1054 * CSizeComputer, which uses the s.seek() method to record bytes that would
1055 * be written instead.
1056 */
1057 class CSizeComputer
1058 {
1059 protected:
1060 size_t nSize;
1061
1062 const int nVersion;
1063 public:
CSizeComputer(int nVersionIn)1064 explicit CSizeComputer(int nVersionIn) : nSize(0), nVersion(nVersionIn) {}
1065
write(const char * psz,size_t _nSize)1066 void write(const char *psz, size_t _nSize)
1067 {
1068 this->nSize += _nSize;
1069 }
1070
1071 /** Pretend _nSize bytes are written, without specifying them. */
seek(size_t _nSize)1072 void seek(size_t _nSize)
1073 {
1074 this->nSize += _nSize;
1075 }
1076
1077 template<typename T>
1078 CSizeComputer& operator<<(const T& obj)
1079 {
1080 ::Serialize(*this, obj);
1081 return (*this);
1082 }
1083
size()1084 size_t size() const {
1085 return nSize;
1086 }
1087
GetVersion()1088 int GetVersion() const { return nVersion; }
1089 };
1090
1091 template<typename Stream>
SerializeMany(Stream & s)1092 void SerializeMany(Stream& s)
1093 {
1094 }
1095
1096 template<typename Stream, typename Arg, typename... Args>
SerializeMany(Stream & s,const Arg & arg,const Args &...args)1097 void SerializeMany(Stream& s, const Arg& arg, const Args&... args)
1098 {
1099 ::Serialize(s, arg);
1100 ::SerializeMany(s, args...);
1101 }
1102
1103 template<typename Stream>
UnserializeMany(Stream & s)1104 inline void UnserializeMany(Stream& s)
1105 {
1106 }
1107
1108 template<typename Stream, typename Arg, typename... Args>
UnserializeMany(Stream & s,Arg && arg,Args &&...args)1109 inline void UnserializeMany(Stream& s, Arg&& arg, Args&&... args)
1110 {
1111 ::Unserialize(s, arg);
1112 ::UnserializeMany(s, args...);
1113 }
1114
1115 template<typename Stream, typename... Args>
SerReadWriteMany(Stream & s,CSerActionSerialize ser_action,const Args &...args)1116 inline void SerReadWriteMany(Stream& s, CSerActionSerialize ser_action, const Args&... args)
1117 {
1118 ::SerializeMany(s, args...);
1119 }
1120
1121 template<typename Stream, typename... Args>
SerReadWriteMany(Stream & s,CSerActionUnserialize ser_action,Args &&...args)1122 inline void SerReadWriteMany(Stream& s, CSerActionUnserialize ser_action, Args&&... args)
1123 {
1124 ::UnserializeMany(s, args...);
1125 }
1126
1127 template<typename I>
WriteVarInt(CSizeComputer & s,I n)1128 inline void WriteVarInt(CSizeComputer &s, I n)
1129 {
1130 s.seek(GetSizeOfVarInt<I>(n));
1131 }
1132
WriteCompactSize(CSizeComputer & s,uint64_t nSize)1133 inline void WriteCompactSize(CSizeComputer &s, uint64_t nSize)
1134 {
1135 s.seek(GetSizeOfCompactSize(nSize));
1136 }
1137
1138 template <typename T>
1139 size_t GetSerializeSize(const T& t, int nVersion = 0)
1140 {
1141 return (CSizeComputer(nVersion) << t).size();
1142 }
1143
1144 template <typename... T>
GetSerializeSizeMany(int nVersion,const T &...t)1145 size_t GetSerializeSizeMany(int nVersion, const T&... t)
1146 {
1147 CSizeComputer sc(nVersion);
1148 SerializeMany(sc, t...);
1149 return sc.size();
1150 }
1151
1152 #endif // BITCOIN_SERIALIZE_H
1153