Bitcoin ABC  0.26.3
P2P Digital Currency
serialize.h
Go to the documentation of this file.
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2016 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 
6 #ifndef BITCOIN_SERIALIZE_H
7 #define BITCOIN_SERIALIZE_H
8 
9 #include <compat/endian.h>
10 #include <prevector.h>
11 #include <rcu.h>
12 #include <span.h>
13 
14 #include <algorithm>
15 #include <array>
16 #include <cstdint>
17 #include <cstring>
18 #include <ios>
19 #include <limits>
20 #include <map>
21 #include <memory>
22 #include <set>
23 #include <string>
24 #include <utility>
25 #include <vector>
26 
31 static constexpr uint64_t MAX_SIZE = 0x02000000;
32 
37 static const unsigned int MAX_VECTOR_ALLOCATE = 5000000;
38 
49 struct deserialize_type {};
51 
53 inline char *CharCast(char *c) {
54  return c;
55 }
56 inline char *CharCast(uint8_t *c) {
57  return (char *)c;
58 }
59 inline const char *CharCast(const char *c) {
60  return c;
61 }
62 inline const char *CharCast(const uint8_t *c) {
63  return (const char *)c;
64 }
65 
70 template <typename Stream> inline void ser_writedata8(Stream &s, uint8_t obj) {
71  s.write((char *)&obj, 1);
72 }
73 template <typename Stream>
74 inline void ser_writedata16(Stream &s, uint16_t obj) {
75  obj = htole16(obj);
76  s.write((char *)&obj, 2);
77 }
78 template <typename Stream>
79 inline void ser_writedata16be(Stream &s, uint16_t obj) {
80  obj = htobe16(obj);
81  s.write((char *)&obj, 2);
82 }
83 template <typename Stream>
84 inline void ser_writedata32(Stream &s, uint32_t obj) {
85  obj = htole32(obj);
86  s.write((char *)&obj, 4);
87 }
88 template <typename Stream>
89 inline void ser_writedata32be(Stream &s, uint32_t obj) {
90  obj = htobe32(obj);
91  s.write((char *)&obj, 4);
92 }
93 template <typename Stream>
94 inline void ser_writedata64(Stream &s, uint64_t obj) {
95  obj = htole64(obj);
96  s.write((char *)&obj, 8);
97 }
98 template <typename Stream> inline uint8_t ser_readdata8(Stream &s) {
99  uint8_t obj;
100  s.read((char *)&obj, 1);
101  return obj;
102 }
103 template <typename Stream> inline uint16_t ser_readdata16(Stream &s) {
104  uint16_t obj;
105  s.read((char *)&obj, 2);
106  return le16toh(obj);
107 }
108 template <typename Stream> inline uint16_t ser_readdata16be(Stream &s) {
109  uint16_t obj;
110  s.read((char *)&obj, 2);
111  return be16toh(obj);
112 }
113 template <typename Stream> inline uint32_t ser_readdata32(Stream &s) {
114  uint32_t obj;
115  s.read((char *)&obj, 4);
116  return le32toh(obj);
117 }
118 template <typename Stream> inline uint32_t ser_readdata32be(Stream &s) {
119  uint32_t obj;
120  s.read((char *)&obj, 4);
121  return be32toh(obj);
122 }
123 template <typename Stream> inline uint64_t ser_readdata64(Stream &s) {
124  uint64_t obj;
125  s.read((char *)&obj, 8);
126  return le64toh(obj);
127 }
128 inline uint64_t ser_double_to_uint64(double x) {
129  uint64_t tmp;
130  std::memcpy(&tmp, &x, sizeof(x));
131  static_assert(sizeof(tmp) == sizeof(x),
132  "double and uint64_t assumed to have the same size");
133  return tmp;
134 }
135 inline uint32_t ser_float_to_uint32(float x) {
136  uint32_t tmp;
137  std::memcpy(&tmp, &x, sizeof(x));
138  static_assert(sizeof(tmp) == sizeof(x),
139  "float and uint32_t assumed to have the same size");
140  return tmp;
141 }
142 inline double ser_uint64_to_double(uint64_t y) {
143  double tmp;
144  std::memcpy(&tmp, &y, sizeof(y));
145  static_assert(sizeof(tmp) == sizeof(y),
146  "double and uint64_t assumed to have the same size");
147  return tmp;
148 }
149 inline float ser_uint32_to_float(uint32_t y) {
150  float tmp;
151  std::memcpy(&tmp, &y, sizeof(y));
152  static_assert(sizeof(tmp) == sizeof(y),
153  "float and uint32_t assumed to have the same size");
154  return tmp;
155 }
156 
158 //
159 // Templates for serializing to anything that looks like a stream,
160 // i.e. anything that supports .read(char*, size_t) and .write(char*, size_t)
161 //
162 class CSizeComputer;
163 
164 enum {
165  // primary actions
166  SER_NETWORK = (1 << 0),
167  SER_DISK = (1 << 1),
168  SER_GETHASH = (1 << 2),
169 };
170 
173 template <typename X> X &ReadWriteAsHelper(X &x) {
174  return x;
175 }
176 template <typename X> const X &ReadWriteAsHelper(const X &x) {
177  return x;
178 }
179 
180 #define READWRITE(...) (::SerReadWriteMany(s, ser_action, __VA_ARGS__))
181 #define READWRITEAS(type, obj) \
182  (::SerReadWriteMany(s, ser_action, ReadWriteAsHelper<type>(obj)))
183 #define SER_READ(obj, code) \
184  ::SerRead( \
185  s, ser_action, obj, \
186  [&](Stream &s, typename std::remove_const<Type>::type &obj) { code; })
187 #define SER_WRITE(obj, code) \
188  ::SerWrite(s, ser_action, obj, [&](Stream &s, const Type &obj) { code; })
189 
208 #define FORMATTER_METHODS(cls, obj) \
209  template <typename Stream> static void Ser(Stream &s, const cls &obj) { \
210  SerializationOps(obj, s, CSerActionSerialize()); \
211  } \
212  template <typename Stream> static void Unser(Stream &s, cls &obj) { \
213  SerializationOps(obj, s, CSerActionUnserialize()); \
214  } \
215  template <typename Stream, typename Type, typename Operation> \
216  static inline void SerializationOps(Type &obj, Stream &s, \
217  Operation ser_action)
218 
227 #define SERIALIZE_METHODS(cls, obj) \
228  template <typename Stream> void Serialize(Stream &s) const { \
229  static_assert(std::is_same<const cls &, decltype(*this)>::value, \
230  "Serialize type mismatch"); \
231  Ser(s, *this); \
232  } \
233  template <typename Stream> void Unserialize(Stream &s) { \
234  static_assert(std::is_same<cls &, decltype(*this)>::value, \
235  "Unserialize type mismatch"); \
236  Unser(s, *this); \
237  } \
238  FORMATTER_METHODS(cls, obj)
239 
240 #ifndef CHAR_EQUALS_INT8
241 // TODO Get rid of bare char
242 template <typename Stream> inline void Serialize(Stream &s, char a) {
243  ser_writedata8(s, a);
244 }
245 #endif
246 template <typename Stream> inline void Serialize(Stream &s, int8_t a) {
247  ser_writedata8(s, a);
248 }
249 template <typename Stream> inline void Serialize(Stream &s, uint8_t a) {
250  ser_writedata8(s, a);
251 }
252 template <typename Stream> inline void Serialize(Stream &s, int16_t a) {
253  ser_writedata16(s, a);
254 }
255 template <typename Stream> inline void Serialize(Stream &s, uint16_t a) {
256  ser_writedata16(s, a);
257 }
258 template <typename Stream> inline void Serialize(Stream &s, int32_t a) {
259  ser_writedata32(s, a);
260 }
261 template <typename Stream> inline void Serialize(Stream &s, uint32_t a) {
262  ser_writedata32(s, a);
263 }
264 template <typename Stream> inline void Serialize(Stream &s, int64_t a) {
265  ser_writedata64(s, a);
266 }
267 template <typename Stream> inline void Serialize(Stream &s, uint64_t a) {
268  ser_writedata64(s, a);
269 }
270 template <typename Stream> inline void Serialize(Stream &s, float a) {
272 }
273 template <typename Stream> inline void Serialize(Stream &s, double a) {
275 }
276 template <typename Stream, size_t N>
277 inline void Serialize(Stream &s, const int8_t (&a)[N]) {
278  s.write(a, N);
279 }
280 template <typename Stream, size_t N>
281 inline void Serialize(Stream &s, const uint8_t (&a)[N]) {
282  s.write(CharCast(a), N);
283 }
284 template <typename Stream, size_t N>
285 inline void Serialize(Stream &s, const std::array<int8_t, N> &a) {
286  s.write(a.data(), N);
287 }
288 template <typename Stream, size_t N>
289 inline void Serialize(Stream &s, const std::array<uint8_t, N> &a) {
290  s.write(CharCast(a.data()), N);
291 }
292 #ifndef CHAR_EQUALS_INT8
293 // TODO Get rid of bare char
294 template <typename Stream> inline void Unserialize(Stream &s, char &a) {
295  a = ser_readdata8(s);
296 }
297 template <typename Stream, size_t N>
298 inline void Serialize(Stream &s, const char (&a)[N]) {
299  s.write(a, N);
300 }
301 template <typename Stream, size_t N>
302 inline void Serialize(Stream &s, const std::array<char, N> &a) {
303  s.write(a.data(), N);
304 }
305 #endif
306 template <typename Stream>
307 inline void Serialize(Stream &s, const Span<const uint8_t> &span) {
308  s.write(CharCast(span.data()), span.size());
309 }
310 template <typename Stream>
311 inline void Serialize(Stream &s, const Span<uint8_t> &span) {
312  s.write(CharCast(span.data()), span.size());
313 }
314 template <typename Stream> inline void Unserialize(Stream &s, int8_t &a) {
315  a = ser_readdata8(s);
316 }
317 template <typename Stream> inline void Unserialize(Stream &s, uint8_t &a) {
318  a = ser_readdata8(s);
319 }
320 template <typename Stream> inline void Unserialize(Stream &s, int16_t &a) {
321  a = ser_readdata16(s);
322 }
323 template <typename Stream> inline void Unserialize(Stream &s, uint16_t &a) {
324  a = ser_readdata16(s);
325 }
326 template <typename Stream> inline void Unserialize(Stream &s, int32_t &a) {
327  a = ser_readdata32(s);
328 }
329 template <typename Stream> inline void Unserialize(Stream &s, uint32_t &a) {
330  a = ser_readdata32(s);
331 }
332 template <typename Stream> inline void Unserialize(Stream &s, int64_t &a) {
333  a = ser_readdata64(s);
334 }
335 template <typename Stream> inline void Unserialize(Stream &s, uint64_t &a) {
336  a = ser_readdata64(s);
337 }
338 template <typename Stream> inline void Unserialize(Stream &s, float &a) {
340 }
341 template <typename Stream> inline void Unserialize(Stream &s, double &a) {
343 }
344 template <typename Stream, size_t N>
345 inline void Unserialize(Stream &s, int8_t (&a)[N]) {
346  s.read(a, N);
347 }
348 template <typename Stream, size_t N>
349 inline void Unserialize(Stream &s, uint8_t (&a)[N]) {
350  s.read(CharCast(a), N);
351 }
352 template <typename Stream, size_t N>
353 inline void Unserialize(Stream &s, std::array<int8_t, N> &a) {
354  s.read(a.data(), N);
355 }
356 template <typename Stream, size_t N>
357 inline void Unserialize(Stream &s, std::array<uint8_t, N> &a) {
358  s.read(CharCast(a.data()), N);
359 }
360 #ifndef CHAR_EQUALS_INT8
361 template <typename Stream, size_t N>
362 inline void Unserialize(Stream &s, char (&a)[N]) {
363  s.read(CharCast(a), N);
364 }
365 template <typename Stream, size_t N>
366 inline void Unserialize(Stream &s, std::array<char, N> &a) {
367  s.read(CharCast(a.data()), N);
368 }
369 #endif
370 
371 template <typename Stream> inline void Serialize(Stream &s, bool a) {
372  char f = a;
373  ser_writedata8(s, f);
374 }
375 template <typename Stream> inline void Unserialize(Stream &s, bool &a) {
376  char f = ser_readdata8(s);
377  a = f;
378 }
379 template <typename Stream>
380 inline void Unserialize(Stream &s, Span<uint8_t> &span) {
381  s.read(CharCast(span.data()), span.size());
382 }
383 
391 inline uint32_t GetSizeOfCompactSize(uint64_t nSize) {
392  if (nSize < 253) {
393  return sizeof(uint8_t);
394  }
395  if (nSize <= std::numeric_limits<uint16_t>::max()) {
396  return sizeof(uint8_t) + sizeof(uint16_t);
397  }
398  if (nSize <= std::numeric_limits<uint32_t>::max()) {
399  return sizeof(uint8_t) + sizeof(uint32_t);
400  }
401 
402  return sizeof(uint8_t) + sizeof(uint64_t);
403 }
404 
405 inline void WriteCompactSize(CSizeComputer &os, uint64_t nSize);
406 
407 template <typename Stream> void WriteCompactSize(Stream &os, uint64_t nSize) {
408  if (nSize < 253) {
409  ser_writedata8(os, nSize);
410  } else if (nSize <= std::numeric_limits<uint16_t>::max()) {
411  ser_writedata8(os, 253);
412  ser_writedata16(os, nSize);
413  } else if (nSize <= std::numeric_limits<uint32_t>::max()) {
414  ser_writedata8(os, 254);
415  ser_writedata32(os, nSize);
416  } else {
417  ser_writedata8(os, 255);
418  ser_writedata64(os, nSize);
419  }
420  return;
421 }
422 
430 template <typename Stream>
431 uint64_t ReadCompactSize(Stream &is, bool range_check = true) {
432  uint8_t chSize = ser_readdata8(is);
433  uint64_t nSizeRet = 0;
434  if (chSize < 253) {
435  nSizeRet = chSize;
436  } else if (chSize == 253) {
437  nSizeRet = ser_readdata16(is);
438  if (nSizeRet < 253) {
439  throw std::ios_base::failure("non-canonical ReadCompactSize()");
440  }
441  } else if (chSize == 254) {
442  nSizeRet = ser_readdata32(is);
443  if (nSizeRet < 0x10000u) {
444  throw std::ios_base::failure("non-canonical ReadCompactSize()");
445  }
446  } else {
447  nSizeRet = ser_readdata64(is);
448  if (nSizeRet < 0x100000000ULL) {
449  throw std::ios_base::failure("non-canonical ReadCompactSize()");
450  }
451  }
452  if (range_check && nSizeRet > MAX_SIZE) {
453  throw std::ios_base::failure("ReadCompactSize(): size too large");
454  }
455  return nSizeRet;
456 }
457 
493 
494 template <VarIntMode Mode, typename I> struct CheckVarIntMode {
495  constexpr CheckVarIntMode() {
496  static_assert(Mode != VarIntMode::DEFAULT || std::is_unsigned<I>::value,
497  "Unsigned type required with mode DEFAULT.");
498  static_assert(Mode != VarIntMode::NONNEGATIVE_SIGNED ||
499  std::is_signed<I>::value,
500  "Signed type required with mode NONNEGATIVE_SIGNED.");
501  }
502 };
503 
504 template <VarIntMode Mode, typename I>
505 inline unsigned int GetSizeOfVarInt(I n) {
507  int nRet = 0;
508  while (true) {
509  nRet++;
510  if (n <= 0x7F) {
511  return nRet;
512  }
513  n = (n >> 7) - 1;
514  }
515 }
516 
517 template <typename I> inline void WriteVarInt(CSizeComputer &os, I n);
518 
519 template <typename Stream, VarIntMode Mode, typename I>
520 void WriteVarInt(Stream &os, I n) {
522  uint8_t tmp[(sizeof(n) * 8 + 6) / 7];
523  int len = 0;
524  while (true) {
525  tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00);
526  if (n <= 0x7F) {
527  break;
528  }
529  n = (n >> 7) - 1;
530  len++;
531  }
532  do {
533  ser_writedata8(os, tmp[len]);
534  } while (len--);
535 }
536 
537 template <typename Stream, VarIntMode Mode, typename I>
538 I ReadVarInt(Stream &is) {
540  I n = 0;
541  while (true) {
542  uint8_t chData = ser_readdata8(is);
543  if (n > (std::numeric_limits<I>::max() >> 7)) {
544  throw std::ios_base::failure("ReadVarInt(): size too large");
545  }
546  n = (n << 7) | (chData & 0x7F);
547  if ((chData & 0x80) == 0) {
548  return n;
549  }
550  if (n == std::numeric_limits<I>::max()) {
551  throw std::ios_base::failure("ReadVarInt(): size too large");
552  }
553  n++;
554  }
555 }
556 
561 template <typename Formatter, typename T> class Wrapper {
562  static_assert(std::is_lvalue_reference<T>::value,
563  "Wrapper needs an lvalue reference type T");
564 
565 protected:
567 
568 public:
569  explicit Wrapper(T obj) : m_object(obj) {}
570  template <typename Stream> void Serialize(Stream &s) const {
571  Formatter().Ser(s, m_object);
572  }
573  template <typename Stream> void Unserialize(Stream &s) {
574  Formatter().Unser(s, m_object);
575  }
576 };
577 
591 template <typename Formatter, typename T>
592 static inline Wrapper<Formatter, T &> Using(T &&t) {
593  return Wrapper<Formatter, T &>(t);
594 }
595 
596 #define VARINT_MODE(obj, mode) Using<VarIntFormatter<mode>>(obj)
597 #define VARINT(obj) Using<VarIntFormatter<VarIntMode::DEFAULT>>(obj)
598 #define COMPACTSIZE(obj) Using<CompactSizeFormatter<true>>(obj)
599 #define LIMITED_STRING(obj, n) Using<LimitedStringFormatter<n>>(obj)
600 
604 template <VarIntMode Mode> struct VarIntFormatter {
605  template <typename Stream, typename I> void Ser(Stream &s, I v) {
606  WriteVarInt<Stream, Mode, typename std::remove_cv<I>::type>(s, v);
607  }
608 
609  template <typename Stream, typename I> void Unser(Stream &s, I &v) {
610  v = ReadVarInt<Stream, Mode, typename std::remove_cv<I>::type>(s);
611  }
612 };
613 
624 template <int Bytes, bool BigEndian = false> struct CustomUintFormatter {
625  static_assert(Bytes > 0 && Bytes <= 8,
626  "CustomUintFormatter Bytes out of range");
627  static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes));
628 
629  template <typename Stream, typename I> void Ser(Stream &s, I v) {
630  if (v < 0 || v > MAX) {
631  throw std::ios_base::failure(
632  "CustomUintFormatter value out of range");
633  }
634  if (BigEndian) {
635  uint64_t raw = htobe64(v);
636  s.write(((const char *)&raw) + 8 - Bytes, Bytes);
637  } else {
638  uint64_t raw = htole64(v);
639  s.write((const char *)&raw, Bytes);
640  }
641  }
642 
643  template <typename Stream, typename I> void Unser(Stream &s, I &v) {
644  using U = typename std::conditional<std::is_enum<I>::value,
645  std::underlying_type<I>,
646  std::common_type<I>>::type::type;
647  static_assert(std::numeric_limits<U>::max() >= MAX &&
648  std::numeric_limits<U>::min() <= 0,
649  "Assigned type too small");
650  uint64_t raw = 0;
651  if (BigEndian) {
652  s.read(((char *)&raw) + 8 - Bytes, Bytes);
653  v = static_cast<I>(be64toh(raw));
654  } else {
655  s.read((char *)&raw, Bytes);
656  v = static_cast<I>(le64toh(raw));
657  }
658  }
659 };
660 
661 template <int Bytes>
663 
665 template <bool RangeCheck> struct CompactSizeFormatter {
666  template <typename Stream, typename I> void Unser(Stream &s, I &v) {
667  uint64_t n = ReadCompactSize<Stream>(s, RangeCheck);
668  if (n < std::numeric_limits<I>::min() ||
669  n > std::numeric_limits<I>::max()) {
670  throw std::ios_base::failure("CompactSize exceeds limit of type");
671  }
672  v = n;
673  }
674 
675  template <typename Stream, typename I> void Ser(Stream &s, I v) {
676  static_assert(std::is_unsigned<I>::value,
677  "CompactSize only supported for unsigned integers");
678  static_assert(std::numeric_limits<I>::max() <=
679  std::numeric_limits<uint64_t>::max(),
680  "CompactSize only supports 64-bit integers and below");
681 
682  WriteCompactSize<Stream>(s, v);
683  }
684 };
685 
686 template <typename U, bool LOSSY = false> struct ChronoFormatter {
687  template <typename Stream, typename Tp> void Unser(Stream &s, Tp &tp) {
688  U u;
689  s >> u;
690  // Lossy deserialization does not make sense, so force Wnarrowing
691  tp = Tp{typename Tp::duration{typename Tp::duration::rep{u}}};
692  }
693  template <typename Stream, typename Tp> void Ser(Stream &s, Tp tp) {
694  if constexpr (LOSSY) {
695  s << U(tp.time_since_epoch().count());
696  } else {
697  s << U{tp.time_since_epoch().count()};
698  }
699  }
700 };
701 template <typename U> using LossyChronoFormatter = ChronoFormatter<U, true>;
702 
703 template <size_t Limit> struct LimitedStringFormatter {
704  template <typename Stream> void Unser(Stream &s, std::string &v) {
705  size_t size = ReadCompactSize(s);
706  if (size > Limit) {
707  throw std::ios_base::failure("String length limit exceeded");
708  }
709  v.resize(size);
710  if (size != 0) {
711  s.read((char *)v.data(), size);
712  }
713  }
714 
715  template <typename Stream> void Ser(Stream &s, const std::string &v) {
716  s << v;
717  }
718 };
719 
736 template <class Formatter> struct VectorFormatter {
737  template <typename Stream, typename V> void Ser(Stream &s, const V &v) {
738  Formatter formatter;
739  WriteCompactSize(s, v.size());
740  for (const typename V::value_type &elem : v) {
741  formatter.Ser(s, elem);
742  }
743  }
744 
745  template <typename Stream, typename V> void Unser(Stream &s, V &v) {
746  Formatter formatter;
747  v.clear();
748  size_t size = ReadCompactSize(s);
749  size_t allocated = 0;
750  while (allocated < size) {
751  // For DoS prevention, do not blindly allocate as much as the stream
752  // claims to contain. Instead, allocate in 5MiB batches, so that an
753  // attacker actually needs to provide X MiB of data to make us
754  // allocate X+5 Mib.
755  static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE,
756  "Vector element size too large");
757  allocated =
758  std::min(size, allocated + MAX_VECTOR_ALLOCATE /
759  sizeof(typename V::value_type));
760  v.reserve(allocated);
761  while (v.size() < allocated) {
762  v.emplace_back();
763  formatter.Unser(s, v.back());
764  }
765  }
766  };
767 };
768 
780  uint64_t m_shift = 0;
781 
782 public:
783  template <typename Stream, typename I> void Ser(Stream &s, I v) {
784  if (v < m_shift || v >= std::numeric_limits<uint64_t>::max()) {
785  throw std::ios_base::failure("differential value overflow");
786  }
787  WriteCompactSize(s, v - m_shift);
788  m_shift = uint64_t(v) + 1;
789  }
790  template <typename Stream, typename I> void Unser(Stream &s, I &v) {
791  uint64_t n = ReadCompactSize(s);
792  m_shift += n;
793  if (m_shift < n || m_shift >= std::numeric_limits<uint64_t>::max() ||
794  m_shift < std::numeric_limits<I>::min() ||
795  m_shift > std::numeric_limits<I>::max()) {
796  throw std::ios_base::failure("differential value overflow");
797  }
798  v = I(m_shift++);
799  }
800 };
801 
813  template <typename Stream, typename T> void Ser(Stream &s, T v) {
814  DifferenceFormatter::Ser(s, v.index);
815  v.SerData(s);
816  }
817 
818  template <typename Stream, typename T> void Unser(Stream &s, T &v) {
819  DifferenceFormatter::Unser(s, v.index);
820  v.UnserData(s);
821  }
822 };
823 
831 template <typename Stream, typename C>
832 void Serialize(Stream &os, const std::basic_string<C> &str);
833 template <typename Stream, typename C>
834 void Unserialize(Stream &is, std::basic_string<C> &str);
835 
841 template <typename Stream, unsigned int N, typename T>
842 void Serialize_impl(Stream &os, const prevector<N, T> &v, const uint8_t &);
843 template <typename Stream, unsigned int N, typename T, typename V>
844 void Serialize_impl(Stream &os, const prevector<N, T> &v, const V &);
845 template <typename Stream, unsigned int N, typename T>
846 inline void Serialize(Stream &os, const prevector<N, T> &v);
847 template <typename Stream, unsigned int N, typename T>
848 void Unserialize_impl(Stream &is, prevector<N, T> &v, const uint8_t &);
849 template <typename Stream, unsigned int N, typename T, typename V>
850 void Unserialize_impl(Stream &is, prevector<N, T> &v, const V &);
851 template <typename Stream, unsigned int N, typename T>
852 inline void Unserialize(Stream &is, prevector<N, T> &v);
853 
859 template <typename Stream, typename T, typename A>
860 void Serialize_impl(Stream &os, const std::vector<T, A> &v, const uint8_t &);
861 template <typename Stream, typename T, typename A>
862 void Serialize_impl(Stream &os, const std::vector<T, A> &v, const bool &);
863 template <typename Stream, typename T, typename A, typename V>
864 void Serialize_impl(Stream &os, const std::vector<T, A> &v, const V &);
865 template <typename Stream, typename T, typename A>
866 inline void Serialize(Stream &os, const std::vector<T, A> &v);
867 template <typename Stream, typename T, typename A>
868 void Unserialize_impl(Stream &is, std::vector<T, A> &v, const uint8_t &);
869 template <typename Stream, typename T, typename A, typename V>
870 void Unserialize_impl(Stream &is, std::vector<T, A> &v, const V &);
871 template <typename Stream, typename T, typename A>
872 inline void Unserialize(Stream &is, std::vector<T, A> &v);
873 
877 template <typename Stream, typename K, typename T>
878 void Serialize(Stream &os, const std::pair<K, T> &item);
879 template <typename Stream, typename K, typename T>
880 void Unserialize(Stream &is, std::pair<K, T> &item);
881 
885 template <typename Stream, typename K, typename T, typename Pred, typename A>
886 void Serialize(Stream &os, const std::map<K, T, Pred, A> &m);
887 template <typename Stream, typename K, typename T, typename Pred, typename A>
888 void Unserialize(Stream &is, std::map<K, T, Pred, A> &m);
889 
893 template <typename Stream, typename K, typename Pred, typename A>
894 void Serialize(Stream &os, const std::set<K, Pred, A> &m);
895 template <typename Stream, typename K, typename Pred, typename A>
896 void Unserialize(Stream &is, std::set<K, Pred, A> &m);
897 
901 template <typename Stream, typename T>
902 void Serialize(Stream &os, const std::shared_ptr<const T> &p);
903 template <typename Stream, typename T>
904 void Unserialize(Stream &os, std::shared_ptr<const T> &p);
905 
909 template <typename Stream, typename T>
910 void Serialize(Stream &os, const std::unique_ptr<const T> &p);
911 template <typename Stream, typename T>
912 void Unserialize(Stream &os, std::unique_ptr<const T> &p);
913 
917 template <typename Stream, typename T>
918 void Serialize(Stream &os, const RCUPtr<const T> &p);
919 template <typename Stream, typename T>
920 void Unserialize(Stream &os, RCUPtr<const T> &p);
921 
926 template <typename Stream, typename T>
927 inline void Serialize(Stream &os, const T &a) {
928  a.Serialize(os);
929 }
930 
931 template <typename Stream, typename T>
932 inline void Unserialize(Stream &is, T &&a) {
933  a.Unserialize(is);
934 }
935 
943  template <typename Stream, typename T>
944  static void Ser(Stream &s, const T &t) {
945  Serialize(s, t);
946  }
947 
948  template <typename Stream, typename T> static void Unser(Stream &s, T &t) {
949  Unserialize(s, t);
950  }
951 };
952 
956 template <typename Stream, typename C>
957 void Serialize(Stream &os, const std::basic_string<C> &str) {
958  WriteCompactSize(os, str.size());
959  if (!str.empty()) {
960  os.write((char *)str.data(), str.size() * sizeof(C));
961  }
962 }
963 
964 template <typename Stream, typename C>
965 void Unserialize(Stream &is, std::basic_string<C> &str) {
966  size_t nSize = ReadCompactSize(is);
967  str.resize(nSize);
968  if (nSize != 0) {
969  is.read((char *)str.data(), nSize * sizeof(C));
970  }
971 }
972 
976 template <typename Stream, unsigned int N, typename T>
977 void Serialize_impl(Stream &os, const prevector<N, T> &v, const uint8_t &) {
978  WriteCompactSize(os, v.size());
979  if (!v.empty()) {
980  os.write((char *)v.data(), v.size() * sizeof(T));
981  }
982 }
983 
984 template <typename Stream, unsigned int N, typename T, typename V>
985 void Serialize_impl(Stream &os, const prevector<N, T> &v, const V &) {
987 }
988 
989 template <typename Stream, unsigned int N, typename T>
990 inline void Serialize(Stream &os, const prevector<N, T> &v) {
991  Serialize_impl(os, v, T());
992 }
993 
994 template <typename Stream, unsigned int N, typename T>
995 void Unserialize_impl(Stream &is, prevector<N, T> &v, const uint8_t &) {
996  // Limit size per read so bogus size value won't cause out of memory
997  v.clear();
998  size_t nSize = ReadCompactSize(is);
999  size_t i = 0;
1000  while (i < nSize) {
1001  size_t blk = std::min(nSize - i, size_t(1 + 4999999 / sizeof(T)));
1002  v.resize_uninitialized(i + blk);
1003  is.read((char *)&v[i], blk * sizeof(T));
1004  i += blk;
1005  }
1006 }
1007 
1008 template <typename Stream, unsigned int N, typename T, typename V>
1009 void Unserialize_impl(Stream &is, prevector<N, T> &v, const V &) {
1011 }
1012 
1013 template <typename Stream, unsigned int N, typename T>
1014 inline void Unserialize(Stream &is, prevector<N, T> &v) {
1015  Unserialize_impl(is, v, T());
1016 }
1017 
1021 template <typename Stream, typename T, typename A>
1022 void Serialize_impl(Stream &os, const std::vector<T, A> &v, const uint8_t &) {
1023  WriteCompactSize(os, v.size());
1024  if (!v.empty()) {
1025  os.write((char *)v.data(), v.size() * sizeof(T));
1026  }
1027 }
1028 
1029 template <typename Stream, typename T, typename A>
1030 void Serialize_impl(Stream &os, const std::vector<T, A> &v, const bool &) {
1031  // A special case for std::vector<bool>, as dereferencing
1032  // std::vector<bool>::const_iterator does not result in a const bool&
1033  // due to std::vector's special casing for bool arguments.
1034  WriteCompactSize(os, v.size());
1035  for (bool elem : v) {
1036  ::Serialize(os, elem);
1037  }
1038 }
1039 
1040 template <typename Stream, typename T, typename A, typename V>
1041 void Serialize_impl(Stream &os, const std::vector<T, A> &v, const V &) {
1043 }
1044 
1045 template <typename Stream, typename T, typename A>
1046 inline void Serialize(Stream &os, const std::vector<T, A> &v) {
1047  Serialize_impl(os, v, T());
1048 }
1049 
1050 template <typename Stream, typename T, typename A>
1051 void Unserialize_impl(Stream &is, std::vector<T, A> &v, const uint8_t &) {
1052  // Limit size per read so bogus size value won't cause out of memory
1053  v.clear();
1054  size_t nSize = ReadCompactSize(is);
1055  size_t i = 0;
1056  while (i < nSize) {
1057  size_t blk = std::min(nSize - i, size_t(1 + 4999999 / sizeof(T)));
1058  v.resize(i + blk);
1059  is.read((char *)&v[i], blk * sizeof(T));
1060  i += blk;
1061  }
1062 }
1063 
1064 template <typename Stream, typename T, typename A, typename V>
1065 void Unserialize_impl(Stream &is, std::vector<T, A> &v, const V &) {
1067 }
1068 
1069 template <typename Stream, typename T, typename A>
1070 inline void Unserialize(Stream &is, std::vector<T, A> &v) {
1071  Unserialize_impl(is, v, T());
1072 }
1073 
1077 template <typename Stream, typename K, typename T>
1078 void Serialize(Stream &os, const std::pair<K, T> &item) {
1079  Serialize(os, item.first);
1080  Serialize(os, item.second);
1081 }
1082 
1083 template <typename Stream, typename K, typename T>
1084 void Unserialize(Stream &is, std::pair<K, T> &item) {
1085  Unserialize(is, item.first);
1086  Unserialize(is, item.second);
1087 }
1088 
1092 template <typename Stream, typename K, typename T, typename Pred, typename A>
1093 void Serialize(Stream &os, const std::map<K, T, Pred, A> &m) {
1094  WriteCompactSize(os, m.size());
1095  for (const auto &entry : m) {
1096  Serialize(os, entry);
1097  }
1098 }
1099 
1100 template <typename Stream, typename K, typename T, typename Pred, typename A>
1101 void Unserialize(Stream &is, std::map<K, T, Pred, A> &m) {
1102  m.clear();
1103  size_t nSize = ReadCompactSize(is);
1104  typename std::map<K, T, Pred, A>::iterator mi = m.begin();
1105  for (size_t i = 0; i < nSize; i++) {
1106  std::pair<K, T> item;
1107  Unserialize(is, item);
1108  mi = m.insert(mi, item);
1109  }
1110 }
1111 
1115 template <typename Stream, typename K, typename Pred, typename A>
1116 void Serialize(Stream &os, const std::set<K, Pred, A> &m) {
1117  WriteCompactSize(os, m.size());
1118  for (const K &i : m) {
1119  Serialize(os, i);
1120  }
1121 }
1122 
1123 template <typename Stream, typename K, typename Pred, typename A>
1124 void Unserialize(Stream &is, std::set<K, Pred, A> &m) {
1125  m.clear();
1126  size_t nSize = ReadCompactSize(is);
1127  typename std::set<K, Pred, A>::iterator it = m.begin();
1128  for (size_t i = 0; i < nSize; i++) {
1129  K key;
1130  Unserialize(is, key);
1131  it = m.insert(it, key);
1132  }
1133 }
1134 
1138 template <typename Stream, typename T>
1139 void Serialize(Stream &os, const std::unique_ptr<const T> &p) {
1140  Serialize(os, *p);
1141 }
1142 
1143 template <typename Stream, typename T>
1144 void Unserialize(Stream &is, std::unique_ptr<const T> &p) {
1145  p.reset(new T(deserialize, is));
1146 }
1147 
1151 template <typename Stream, typename T>
1152 void Serialize(Stream &os, const std::shared_ptr<const T> &p) {
1153  Serialize(os, *p);
1154 }
1155 
1156 template <typename Stream, typename T>
1157 void Unserialize(Stream &is, std::shared_ptr<const T> &p) {
1158  p = std::make_shared<const T>(deserialize, is);
1159 }
1160 
1164 template <typename Stream, typename T>
1165 void Serialize(Stream &os, const RCUPtr<const T> &p) {
1166  Serialize(os, *p);
1167 }
1168 
1169 template <typename Stream, typename T>
1170 void Unserialize(Stream &is, RCUPtr<const T> &p) {
1172 }
1173 
1178  constexpr bool ForRead() const { return false; }
1179 };
1181  constexpr bool ForRead() const { return true; }
1182 };
1183 
1197 protected:
1198  size_t nSize;
1199 
1200  const int nVersion;
1201 
1202 public:
1203  explicit CSizeComputer(int nVersionIn) : nSize(0), nVersion(nVersionIn) {}
1204 
1205  void write(const char *psz, size_t _nSize) { this->nSize += _nSize; }
1206 
1208  void seek(size_t _nSize) { this->nSize += _nSize; }
1209 
1210  template <typename T> CSizeComputer &operator<<(const T &obj) {
1211  ::Serialize(*this, obj);
1212  return (*this);
1213  }
1214 
1215  size_t size() const { return nSize; }
1216 
1217  int GetVersion() const { return nVersion; }
1218 };
1219 
1220 template <typename Stream> void SerializeMany(Stream &s) {}
1221 
1222 template <typename Stream, typename Arg, typename... Args>
1223 void SerializeMany(Stream &s, const Arg &arg, const Args &...args) {
1224  ::Serialize(s, arg);
1225  ::SerializeMany(s, args...);
1226 }
1227 
1228 template <typename Stream> inline void UnserializeMany(Stream &s) {}
1229 
1230 template <typename Stream, typename Arg, typename... Args>
1231 inline void UnserializeMany(Stream &s, Arg &&arg, Args &&...args) {
1232  ::Unserialize(s, arg);
1233  ::UnserializeMany(s, args...);
1234 }
1235 
1236 template <typename Stream, typename... Args>
1237 inline void SerReadWriteMany(Stream &s, CSerActionSerialize ser_action,
1238  const Args &...args) {
1239  ::SerializeMany(s, args...);
1240 }
1241 
1242 template <typename Stream, typename... Args>
1243 inline void SerReadWriteMany(Stream &s, CSerActionUnserialize ser_action,
1244  Args &&...args) {
1245  ::UnserializeMany(s, args...);
1246 }
1247 
1248 template <typename Stream, typename Type, typename Fn>
1249 inline void SerRead(Stream &s, CSerActionSerialize ser_action, Type &&, Fn &&) {
1250 }
1251 
1252 template <typename Stream, typename Type, typename Fn>
1253 inline void SerRead(Stream &s, CSerActionUnserialize ser_action, Type &&obj,
1254  Fn &&fn) {
1255  fn(s, std::forward<Type>(obj));
1256 }
1257 
1258 template <typename Stream, typename Type, typename Fn>
1259 inline void SerWrite(Stream &s, CSerActionSerialize ser_action, Type &&obj,
1260  Fn &&fn) {
1261  fn(s, std::forward<Type>(obj));
1262 }
1263 
1264 template <typename Stream, typename Type, typename Fn>
1265 inline void SerWrite(Stream &s, CSerActionUnserialize ser_action, Type &&,
1266  Fn &&) {}
1267 
1268 template <typename I> inline void WriteVarInt(CSizeComputer &s, I n) {
1269  s.seek(GetSizeOfVarInt<I>(n));
1270 }
1271 
1272 inline void WriteCompactSize(CSizeComputer &s, uint64_t nSize) {
1273  s.seek(GetSizeOfCompactSize(nSize));
1274 }
1275 
1276 template <typename T> size_t GetSerializeSize(const T &t, int nVersion = 0) {
1277  return (CSizeComputer(nVersion) << t).size();
1278 }
1279 
1280 template <typename... T>
1281 size_t GetSerializeSizeMany(int nVersion, const T &...t) {
1282  CSizeComputer sc(nVersion);
1283  SerializeMany(sc, t...);
1284  return sc.size();
1285 }
1286 
1287 #endif // BITCOIN_SERIALIZE_H
GetSerializeSize implementations
Definition: serialize.h:1196
CSizeComputer & operator<<(const T &obj)
Definition: serialize.h:1210
void write(const char *psz, size_t _nSize)
Definition: serialize.h:1205
CSizeComputer(int nVersionIn)
Definition: serialize.h:1203
size_t nSize
Definition: serialize.h:1198
size_t size() const
Definition: serialize.h:1215
const int nVersion
Definition: serialize.h:1200
void seek(size_t _nSize)
Pretend _nSize bytes are written, without specifying them.
Definition: serialize.h:1208
int GetVersion() const
Definition: serialize.h:1217
Helper for differentially encoded Compact Size integers in lists.
Definition: serialize.h:779
void Unser(Stream &s, I &v)
Definition: serialize.h:790
void Ser(Stream &s, I v)
Definition: serialize.h:783
Definition: rcu.h:85
static RCUPtr make(Args &&...args)
Construct a new object that is owned by the pointer.
Definition: rcu.h:112
A Span is an object that can refer to a contiguous sequence of objects.
Definition: span.h:93
constexpr std::size_t size() const noexcept
Definition: span.h:209
constexpr C * data() const noexcept
Definition: span.h:198
Simple wrapper class to serialize objects using a formatter; used by Using().
Definition: serialize.h:561
Wrapper(T obj)
Definition: serialize.h:569
T m_object
Definition: serialize.h:563
void Serialize(Stream &s) const
Definition: serialize.h:570
void Unserialize(Stream &s)
Definition: serialize.h:573
Implements a drop-in replacement for std::vector<T> which stores up to N elements directly (without h...
Definition: prevector.h:38
bool empty() const
Definition: prevector.h:388
void clear()
Definition: prevector.h:441
size_type size() const
Definition: prevector.h:386
value_type * data()
Definition: prevector.h:610
void resize_uninitialized(size_type new_size)
Definition: prevector.h:482
uint16_t be16toh(uint16_t big_endian_16bits)
Definition: endian.h:156
uint16_t htobe16(uint16_t host_16bits)
Definition: endian.h:144
uint32_t le32toh(uint32_t little_endian_32bits)
Definition: endian.h:186
uint32_t htobe32(uint32_t host_32bits)
Definition: endian.h:168
uint16_t le16toh(uint16_t little_endian_16bits)
Definition: endian.h:162
uint64_t htobe64(uint64_t host_64bits)
Definition: endian.h:192
uint64_t be64toh(uint64_t big_endian_64bits)
Definition: endian.h:204
uint32_t be32toh(uint32_t big_endian_32bits)
Definition: endian.h:180
uint64_t htole64(uint64_t host_64bits)
Definition: endian.h:198
uint32_t htole32(uint32_t host_32bits)
Definition: endian.h:174
uint16_t htole16(uint16_t host_16bits)
Definition: endian.h:150
uint64_t le64toh(uint64_t little_endian_64bits)
Definition: endian.h:210
void SerializeMany(Stream &s)
Definition: serialize.h:1220
static const unsigned int MAX_VECTOR_ALLOCATE
Maximum amount of memory (in bytes) to allocate at once when deserializing vectors.
Definition: serialize.h:37
uint8_t ser_readdata8(Stream &s)
Definition: serialize.h:98
float ser_uint32_to_float(uint32_t y)
Definition: serialize.h:149
I ReadVarInt(Stream &is)
Definition: serialize.h:538
void ser_writedata32be(Stream &s, uint32_t obj)
Definition: serialize.h:89
void WriteVarInt(CSizeComputer &os, I n)
Definition: serialize.h:1268
VarIntMode
Variable-length integers: bytes are a MSB base-128 encoding of the number.
Definition: serialize.h:492
@ NONNEGATIVE_SIGNED
void ser_writedata32(Stream &s, uint32_t obj)
Definition: serialize.h:84
size_t GetSerializeSizeMany(int nVersion, const T &...t)
Definition: serialize.h:1281
uint32_t GetSizeOfCompactSize(uint64_t nSize)
Compact Size size < 253 – 1 byte size <= USHRT_MAX – 3 bytes (253 + 2 bytes) size <= UINT_MAX – 5 byt...
Definition: serialize.h:391
static constexpr uint64_t MAX_SIZE
The maximum size of a serialized object in bytes or number of elements (for eg vectors) when the size...
Definition: serialize.h:31
constexpr deserialize_type deserialize
Definition: serialize.h:50
void Serialize(Stream &s, char a)
Definition: serialize.h:242
void ser_writedata16(Stream &s, uint16_t obj)
Definition: serialize.h:74
uint32_t ser_float_to_uint32(float x)
Definition: serialize.h:135
static Wrapper< Formatter, T & > Using(T &&t)
Cause serialization/deserialization of an object to be done using a specified formatter class.
Definition: serialize.h:592
X & ReadWriteAsHelper(X &x)
Convert the reference base type to X, without changing constness or reference type.
Definition: serialize.h:173
@ SER_DISK
Definition: serialize.h:167
@ SER_NETWORK
Definition: serialize.h:166
@ SER_GETHASH
Definition: serialize.h:168
void Unserialize_impl(Stream &is, prevector< N, T > &v, const uint8_t &)
Definition: serialize.h:995
void Unserialize(Stream &s, char &a)
Definition: serialize.h:294
void SerReadWriteMany(Stream &s, CSerActionSerialize ser_action, const Args &...args)
Definition: serialize.h:1237
void SerWrite(Stream &s, CSerActionSerialize ser_action, Type &&obj, Fn &&fn)
Definition: serialize.h:1259
char * CharCast(char *c)
Safely convert odd char pointer types to standard ones.
Definition: serialize.h:53
void ser_writedata16be(Stream &s, uint16_t obj)
Definition: serialize.h:79
uint16_t ser_readdata16(Stream &s)
Definition: serialize.h:103
uint64_t ser_readdata64(Stream &s)
Definition: serialize.h:123
double ser_uint64_to_double(uint64_t y)
Definition: serialize.h:142
void ser_writedata8(Stream &s, uint8_t obj)
Lowest-level serialization and conversion.
Definition: serialize.h:70
uint64_t ReadCompactSize(Stream &is, bool range_check=true)
Decode a CompactSize-encoded variable-length integer.
Definition: serialize.h:431
unsigned int GetSizeOfVarInt(I n)
Definition: serialize.h:505
uint32_t ser_readdata32(Stream &s)
Definition: serialize.h:113
uint16_t ser_readdata16be(Stream &s)
Definition: serialize.h:108
void SerRead(Stream &s, CSerActionSerialize ser_action, Type &&, Fn &&)
Definition: serialize.h:1249
void Serialize_impl(Stream &os, const prevector< N, T > &v, const uint8_t &)
prevector prevectors of uint8_t are a special case and are intended to be serialized as a single opaq...
Definition: serialize.h:977
size_t GetSerializeSize(const T &t, int nVersion=0)
Definition: serialize.h:1276
void UnserializeMany(Stream &s)
Definition: serialize.h:1228
uint64_t ser_double_to_uint64(double x)
Definition: serialize.h:128
void ser_writedata64(Stream &s, uint64_t obj)
Definition: serialize.h:94
uint32_t ser_readdata32be(Stream &s)
Definition: serialize.h:118
void WriteCompactSize(CSizeComputer &os, uint64_t nSize)
Definition: serialize.h:1272
Support for SERIALIZE_METHODS and READWRITE macro.
Definition: serialize.h:1177
constexpr bool ForRead() const
Definition: serialize.h:1178
constexpr bool ForRead() const
Definition: serialize.h:1181
constexpr CheckVarIntMode()
Definition: serialize.h:495
void Unser(Stream &s, Tp &tp)
Definition: serialize.h:687
void Ser(Stream &s, Tp tp)
Definition: serialize.h:693
Formatter for integers in CompactSize format.
Definition: serialize.h:665
void Unser(Stream &s, I &v)
Definition: serialize.h:666
void Ser(Stream &s, I v)
Definition: serialize.h:675
Serialization wrapper class for custom integers and enums.
Definition: serialize.h:624
static constexpr uint64_t MAX
Definition: serialize.h:627
void Ser(Stream &s, I v)
Definition: serialize.h:629
void Unser(Stream &s, I &v)
Definition: serialize.h:643
Default formatter.
Definition: serialize.h:942
static void Ser(Stream &s, const T &t)
Definition: serialize.h:944
static void Unser(Stream &s, T &t)
Definition: serialize.h:948
Helper for a list of items containing a differentially encoded index as their first member.
Definition: serialize.h:812
void Unser(Stream &s, T &v)
Definition: serialize.h:818
void Ser(Stream &s, T v)
Definition: serialize.h:813
void Unser(Stream &s, std::string &v)
Definition: serialize.h:704
void Ser(Stream &s, const std::string &v)
Definition: serialize.h:715
Serialization wrapper class for integers in VarInt format.
Definition: serialize.h:604
void Ser(Stream &s, I v)
Definition: serialize.h:605
void Unser(Stream &s, I &v)
Definition: serialize.h:609
Formatter to serialize/deserialize vector elements using another formatter.
Definition: serialize.h:736
void Unser(Stream &s, V &v)
Definition: serialize.h:745
void Ser(Stream &s, const V &v)
Definition: serialize.h:737
Dummy data type to identify deserializing constructors.
Definition: serialize.h:49