Bitcoin ABC  0.24.11
P2P Digital Currency
serialize.h
Go to the documentation of this file.
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2016 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 
6 #ifndef BITCOIN_SERIALIZE_H
7 #define BITCOIN_SERIALIZE_H
8 
9 #include <compat/endian.h>
10 #include <prevector.h>
11 #include <span.h>
12 
13 #include <algorithm>
14 #include <array>
15 #include <cstdint>
16 #include <cstring>
17 #include <ios>
18 #include <limits>
19 #include <map>
20 #include <memory>
21 #include <set>
22 #include <string>
23 #include <utility>
24 #include <vector>
25 
30 static constexpr uint64_t MAX_SIZE = 0x02000000;
31 
36 static const unsigned int MAX_VECTOR_ALLOCATE = 5000000;
37 
48 struct deserialize_type {};
50 
52 inline char *CharCast(char *c) {
53  return c;
54 }
55 inline char *CharCast(uint8_t *c) {
56  return (char *)c;
57 }
58 inline const char *CharCast(const char *c) {
59  return c;
60 }
61 inline const char *CharCast(const uint8_t *c) {
62  return (const char *)c;
63 }
64 
69 template <typename Stream> inline void ser_writedata8(Stream &s, uint8_t obj) {
70  s.write((char *)&obj, 1);
71 }
72 template <typename Stream>
73 inline void ser_writedata16(Stream &s, uint16_t obj) {
74  obj = htole16(obj);
75  s.write((char *)&obj, 2);
76 }
77 template <typename Stream>
78 inline void ser_writedata16be(Stream &s, uint16_t obj) {
79  obj = htobe16(obj);
80  s.write((char *)&obj, 2);
81 }
82 template <typename Stream>
83 inline void ser_writedata32(Stream &s, uint32_t obj) {
84  obj = htole32(obj);
85  s.write((char *)&obj, 4);
86 }
87 template <typename Stream>
88 inline void ser_writedata32be(Stream &s, uint32_t obj) {
89  obj = htobe32(obj);
90  s.write((char *)&obj, 4);
91 }
92 template <typename Stream>
93 inline void ser_writedata64(Stream &s, uint64_t obj) {
94  obj = htole64(obj);
95  s.write((char *)&obj, 8);
96 }
97 template <typename Stream> inline uint8_t ser_readdata8(Stream &s) {
98  uint8_t obj;
99  s.read((char *)&obj, 1);
100  return obj;
101 }
102 template <typename Stream> inline uint16_t ser_readdata16(Stream &s) {
103  uint16_t obj;
104  s.read((char *)&obj, 2);
105  return le16toh(obj);
106 }
107 template <typename Stream> inline uint16_t ser_readdata16be(Stream &s) {
108  uint16_t obj;
109  s.read((char *)&obj, 2);
110  return be16toh(obj);
111 }
112 template <typename Stream> inline uint32_t ser_readdata32(Stream &s) {
113  uint32_t obj;
114  s.read((char *)&obj, 4);
115  return le32toh(obj);
116 }
117 template <typename Stream> inline uint32_t ser_readdata32be(Stream &s) {
118  uint32_t obj;
119  s.read((char *)&obj, 4);
120  return be32toh(obj);
121 }
122 template <typename Stream> inline uint64_t ser_readdata64(Stream &s) {
123  uint64_t obj;
124  s.read((char *)&obj, 8);
125  return le64toh(obj);
126 }
127 inline uint64_t ser_double_to_uint64(double x) {
128  uint64_t tmp;
129  std::memcpy(&tmp, &x, sizeof(x));
130  static_assert(sizeof(tmp) == sizeof(x),
131  "double and uint64_t assumed to have the same size");
132  return tmp;
133 }
134 inline uint32_t ser_float_to_uint32(float x) {
135  uint32_t tmp;
136  std::memcpy(&tmp, &x, sizeof(x));
137  static_assert(sizeof(tmp) == sizeof(x),
138  "float and uint32_t assumed to have the same size");
139  return tmp;
140 }
141 inline double ser_uint64_to_double(uint64_t y) {
142  double tmp;
143  std::memcpy(&tmp, &y, sizeof(y));
144  static_assert(sizeof(tmp) == sizeof(y),
145  "double and uint64_t assumed to have the same size");
146  return tmp;
147 }
148 inline float ser_uint32_to_float(uint32_t y) {
149  float tmp;
150  std::memcpy(&tmp, &y, sizeof(y));
151  static_assert(sizeof(tmp) == sizeof(y),
152  "float and uint32_t assumed to have the same size");
153  return tmp;
154 }
155 
157 //
158 // Templates for serializing to anything that looks like a stream,
159 // i.e. anything that supports .read(char*, size_t) and .write(char*, size_t)
160 //
161 class CSizeComputer;
162 
163 enum {
164  // primary actions
165  SER_NETWORK = (1 << 0),
166  SER_DISK = (1 << 1),
167  SER_GETHASH = (1 << 2),
168 };
169 
172 template <typename X> X &ReadWriteAsHelper(X &x) {
173  return x;
174 }
175 template <typename X> const X &ReadWriteAsHelper(const X &x) {
176  return x;
177 }
178 
179 #define READWRITE(...) (::SerReadWriteMany(s, ser_action, __VA_ARGS__))
180 #define READWRITEAS(type, obj) \
181  (::SerReadWriteMany(s, ser_action, ReadWriteAsHelper<type>(obj)))
182 #define SER_READ(obj, code) \
183  ::SerRead( \
184  s, ser_action, obj, \
185  [&](Stream &s, typename std::remove_const<Type>::type &obj) { code; })
186 #define SER_WRITE(obj, code) \
187  ::SerWrite(s, ser_action, obj, [&](Stream &s, const Type &obj) { code; })
188 
207 #define FORMATTER_METHODS(cls, obj) \
208  template <typename Stream> static void Ser(Stream &s, const cls &obj) { \
209  SerializationOps(obj, s, CSerActionSerialize()); \
210  } \
211  template <typename Stream> static void Unser(Stream &s, cls &obj) { \
212  SerializationOps(obj, s, CSerActionUnserialize()); \
213  } \
214  template <typename Stream, typename Type, typename Operation> \
215  static inline void SerializationOps(Type &obj, Stream &s, \
216  Operation ser_action)
217 
226 #define SERIALIZE_METHODS(cls, obj) \
227  template <typename Stream> void Serialize(Stream &s) const { \
228  static_assert(std::is_same<const cls &, decltype(*this)>::value, \
229  "Serialize type mismatch"); \
230  Ser(s, *this); \
231  } \
232  template <typename Stream> void Unserialize(Stream &s) { \
233  static_assert(std::is_same<cls &, decltype(*this)>::value, \
234  "Unserialize type mismatch"); \
235  Unser(s, *this); \
236  } \
237  FORMATTER_METHODS(cls, obj)
238 
239 #ifndef CHAR_EQUALS_INT8
240 // TODO Get rid of bare char
241 template <typename Stream> inline void Serialize(Stream &s, char a) {
242  ser_writedata8(s, a);
243 }
244 #endif
245 template <typename Stream> inline void Serialize(Stream &s, int8_t a) {
246  ser_writedata8(s, a);
247 }
248 template <typename Stream> inline void Serialize(Stream &s, uint8_t a) {
249  ser_writedata8(s, a);
250 }
251 template <typename Stream> inline void Serialize(Stream &s, int16_t a) {
252  ser_writedata16(s, a);
253 }
254 template <typename Stream> inline void Serialize(Stream &s, uint16_t a) {
255  ser_writedata16(s, a);
256 }
257 template <typename Stream> inline void Serialize(Stream &s, int32_t a) {
258  ser_writedata32(s, a);
259 }
260 template <typename Stream> inline void Serialize(Stream &s, uint32_t a) {
261  ser_writedata32(s, a);
262 }
263 template <typename Stream> inline void Serialize(Stream &s, int64_t a) {
264  ser_writedata64(s, a);
265 }
266 template <typename Stream> inline void Serialize(Stream &s, uint64_t a) {
267  ser_writedata64(s, a);
268 }
269 template <typename Stream> inline void Serialize(Stream &s, float a) {
271 }
272 template <typename Stream> inline void Serialize(Stream &s, double a) {
274 }
275 template <typename Stream, size_t N>
276 inline void Serialize(Stream &s, const int8_t (&a)[N]) {
277  s.write(a, N);
278 }
279 template <typename Stream, size_t N>
280 inline void Serialize(Stream &s, const uint8_t (&a)[N]) {
281  s.write(CharCast(a), N);
282 }
283 template <typename Stream, size_t N>
284 inline void Serialize(Stream &s, const std::array<int8_t, N> &a) {
285  s.write(a.data(), N);
286 }
287 template <typename Stream, size_t N>
288 inline void Serialize(Stream &s, const std::array<uint8_t, N> &a) {
289  s.write(CharCast(a.data()), N);
290 }
291 #ifndef CHAR_EQUALS_INT8
292 // TODO Get rid of bare char
293 template <typename Stream> inline void Unserialize(Stream &s, char &a) {
294  a = ser_readdata8(s);
295 }
296 template <typename Stream, size_t N>
297 inline void Serialize(Stream &s, const char (&a)[N]) {
298  s.write(a, N);
299 }
300 template <typename Stream, size_t N>
301 inline void Serialize(Stream &s, const std::array<char, N> &a) {
302  s.write(a.data(), N);
303 }
304 #endif
305 template <typename Stream>
306 inline void Serialize(Stream &s, const Span<const uint8_t> &span) {
307  s.write(CharCast(span.data()), span.size());
308 }
309 template <typename Stream>
310 inline void Serialize(Stream &s, const Span<uint8_t> &span) {
311  s.write(CharCast(span.data()), span.size());
312 }
313 template <typename Stream> inline void Unserialize(Stream &s, int8_t &a) {
314  a = ser_readdata8(s);
315 }
316 template <typename Stream> inline void Unserialize(Stream &s, uint8_t &a) {
317  a = ser_readdata8(s);
318 }
319 template <typename Stream> inline void Unserialize(Stream &s, int16_t &a) {
320  a = ser_readdata16(s);
321 }
322 template <typename Stream> inline void Unserialize(Stream &s, uint16_t &a) {
323  a = ser_readdata16(s);
324 }
325 template <typename Stream> inline void Unserialize(Stream &s, int32_t &a) {
326  a = ser_readdata32(s);
327 }
328 template <typename Stream> inline void Unserialize(Stream &s, uint32_t &a) {
329  a = ser_readdata32(s);
330 }
331 template <typename Stream> inline void Unserialize(Stream &s, int64_t &a) {
332  a = ser_readdata64(s);
333 }
334 template <typename Stream> inline void Unserialize(Stream &s, uint64_t &a) {
335  a = ser_readdata64(s);
336 }
337 template <typename Stream> inline void Unserialize(Stream &s, float &a) {
339 }
340 template <typename Stream> inline void Unserialize(Stream &s, double &a) {
342 }
343 template <typename Stream, size_t N>
344 inline void Unserialize(Stream &s, int8_t (&a)[N]) {
345  s.read(a, N);
346 }
347 template <typename Stream, size_t N>
348 inline void Unserialize(Stream &s, uint8_t (&a)[N]) {
349  s.read(CharCast(a), N);
350 }
351 template <typename Stream, size_t N>
352 inline void Unserialize(Stream &s, std::array<int8_t, N> &a) {
353  s.read(a.data(), N);
354 }
355 template <typename Stream, size_t N>
356 inline void Unserialize(Stream &s, std::array<uint8_t, N> &a) {
357  s.read(CharCast(a.data()), N);
358 }
359 #ifndef CHAR_EQUALS_INT8
360 template <typename Stream, size_t N>
361 inline void Unserialize(Stream &s, char (&a)[N]) {
362  s.read(CharCast(a), N);
363 }
364 template <typename Stream, size_t N>
365 inline void Unserialize(Stream &s, std::array<char, N> &a) {
366  s.read(CharCast(a.data()), N);
367 }
368 #endif
369 
370 template <typename Stream> inline void Serialize(Stream &s, bool a) {
371  char f = a;
372  ser_writedata8(s, f);
373 }
374 template <typename Stream> inline void Unserialize(Stream &s, bool &a) {
375  char f = ser_readdata8(s);
376  a = f;
377 }
378 template <typename Stream>
379 inline void Unserialize(Stream &s, Span<uint8_t> &span) {
380  s.read(CharCast(span.data()), span.size());
381 }
382 
390 inline uint32_t GetSizeOfCompactSize(uint64_t nSize) {
391  if (nSize < 253) {
392  return sizeof(uint8_t);
393  }
394  if (nSize <= std::numeric_limits<uint16_t>::max()) {
395  return sizeof(uint8_t) + sizeof(uint16_t);
396  }
397  if (nSize <= std::numeric_limits<uint32_t>::max()) {
398  return sizeof(uint8_t) + sizeof(uint32_t);
399  }
400 
401  return sizeof(uint8_t) + sizeof(uint64_t);
402 }
403 
404 inline void WriteCompactSize(CSizeComputer &os, uint64_t nSize);
405 
406 template <typename Stream> void WriteCompactSize(Stream &os, uint64_t nSize) {
407  if (nSize < 253) {
408  ser_writedata8(os, nSize);
409  } else if (nSize <= std::numeric_limits<uint16_t>::max()) {
410  ser_writedata8(os, 253);
411  ser_writedata16(os, nSize);
412  } else if (nSize <= std::numeric_limits<uint32_t>::max()) {
413  ser_writedata8(os, 254);
414  ser_writedata32(os, nSize);
415  } else {
416  ser_writedata8(os, 255);
417  ser_writedata64(os, nSize);
418  }
419  return;
420 }
421 
429 template <typename Stream>
430 uint64_t ReadCompactSize(Stream &is, bool range_check = true) {
431  uint8_t chSize = ser_readdata8(is);
432  uint64_t nSizeRet = 0;
433  if (chSize < 253) {
434  nSizeRet = chSize;
435  } else if (chSize == 253) {
436  nSizeRet = ser_readdata16(is);
437  if (nSizeRet < 253) {
438  throw std::ios_base::failure("non-canonical ReadCompactSize()");
439  }
440  } else if (chSize == 254) {
441  nSizeRet = ser_readdata32(is);
442  if (nSizeRet < 0x10000u) {
443  throw std::ios_base::failure("non-canonical ReadCompactSize()");
444  }
445  } else {
446  nSizeRet = ser_readdata64(is);
447  if (nSizeRet < 0x100000000ULL) {
448  throw std::ios_base::failure("non-canonical ReadCompactSize()");
449  }
450  }
451  if (range_check && nSizeRet > MAX_SIZE) {
452  throw std::ios_base::failure("ReadCompactSize(): size too large");
453  }
454  return nSizeRet;
455 }
456 
492 
493 template <VarIntMode Mode, typename I> struct CheckVarIntMode {
494  constexpr CheckVarIntMode() {
495  static_assert(Mode != VarIntMode::DEFAULT || std::is_unsigned<I>::value,
496  "Unsigned type required with mode DEFAULT.");
497  static_assert(Mode != VarIntMode::NONNEGATIVE_SIGNED ||
498  std::is_signed<I>::value,
499  "Signed type required with mode NONNEGATIVE_SIGNED.");
500  }
501 };
502 
503 template <VarIntMode Mode, typename I>
504 inline unsigned int GetSizeOfVarInt(I n) {
506  int nRet = 0;
507  while (true) {
508  nRet++;
509  if (n <= 0x7F) {
510  return nRet;
511  }
512  n = (n >> 7) - 1;
513  }
514 }
515 
516 template <typename I> inline void WriteVarInt(CSizeComputer &os, I n);
517 
518 template <typename Stream, VarIntMode Mode, typename I>
519 void WriteVarInt(Stream &os, I n) {
521  uint8_t tmp[(sizeof(n) * 8 + 6) / 7];
522  int len = 0;
523  while (true) {
524  tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00);
525  if (n <= 0x7F) {
526  break;
527  }
528  n = (n >> 7) - 1;
529  len++;
530  }
531  do {
532  ser_writedata8(os, tmp[len]);
533  } while (len--);
534 }
535 
536 template <typename Stream, VarIntMode Mode, typename I>
537 I ReadVarInt(Stream &is) {
539  I n = 0;
540  while (true) {
541  uint8_t chData = ser_readdata8(is);
542  if (n > (std::numeric_limits<I>::max() >> 7)) {
543  throw std::ios_base::failure("ReadVarInt(): size too large");
544  }
545  n = (n << 7) | (chData & 0x7F);
546  if ((chData & 0x80) == 0) {
547  return n;
548  }
549  if (n == std::numeric_limits<I>::max()) {
550  throw std::ios_base::failure("ReadVarInt(): size too large");
551  }
552  n++;
553  }
554 }
555 
560 template <typename Formatter, typename T> class Wrapper {
561  static_assert(std::is_lvalue_reference<T>::value,
562  "Wrapper needs an lvalue reference type T");
563 
564 protected:
565  T m_object;
566 
567 public:
568  explicit Wrapper(T obj) : m_object(obj) {}
569  template <typename Stream> void Serialize(Stream &s) const {
570  Formatter().Ser(s, m_object);
571  }
572  template <typename Stream> void Unserialize(Stream &s) {
573  Formatter().Unser(s, m_object);
574  }
575 };
576 
590 template <typename Formatter, typename T>
591 static inline Wrapper<Formatter, T &> Using(T &&t) {
592  return Wrapper<Formatter, T &>(t);
593 }
594 
595 #define VARINT_MODE(obj, mode) Using<VarIntFormatter<mode>>(obj)
596 #define VARINT(obj) Using<VarIntFormatter<VarIntMode::DEFAULT>>(obj)
597 #define COMPACTSIZE(obj) Using<CompactSizeFormatter<true>>(obj)
598 #define LIMITED_STRING(obj, n) Using<LimitedStringFormatter<n>>(obj)
599 
603 template <VarIntMode Mode> struct VarIntFormatter {
604  template <typename Stream, typename I> void Ser(Stream &s, I v) {
605  WriteVarInt<Stream, Mode, typename std::remove_cv<I>::type>(s, v);
606  }
607 
608  template <typename Stream, typename I> void Unser(Stream &s, I &v) {
609  v = ReadVarInt<Stream, Mode, typename std::remove_cv<I>::type>(s);
610  }
611 };
612 
623 template <int Bytes, bool BigEndian = false> struct CustomUintFormatter {
624  static_assert(Bytes > 0 && Bytes <= 8,
625  "CustomUintFormatter Bytes out of range");
626  static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes));
627 
628  template <typename Stream, typename I> void Ser(Stream &s, I v) {
629  if (v < 0 || v > MAX) {
630  throw std::ios_base::failure(
631  "CustomUintFormatter value out of range");
632  }
633  if (BigEndian) {
634  uint64_t raw = htobe64(v);
635  s.write(((const char *)&raw) + 8 - Bytes, Bytes);
636  } else {
637  uint64_t raw = htole64(v);
638  s.write((const char *)&raw, Bytes);
639  }
640  }
641 
642  template <typename Stream, typename I> void Unser(Stream &s, I &v) {
643  using U = typename std::conditional<std::is_enum<I>::value,
644  std::underlying_type<I>,
645  std::common_type<I>>::type::type;
646  static_assert(std::numeric_limits<U>::max() >= MAX &&
647  std::numeric_limits<U>::min() <= 0,
648  "Assigned type too small");
649  uint64_t raw = 0;
650  if (BigEndian) {
651  s.read(((char *)&raw) + 8 - Bytes, Bytes);
652  v = static_cast<I>(be64toh(raw));
653  } else {
654  s.read((char *)&raw, Bytes);
655  v = static_cast<I>(le64toh(raw));
656  }
657  }
658 };
659 
660 template <int Bytes>
662 
664 template <bool RangeCheck> struct CompactSizeFormatter {
665  template <typename Stream, typename I> void Unser(Stream &s, I &v) {
666  uint64_t n = ReadCompactSize<Stream>(s, RangeCheck);
667  if (n < std::numeric_limits<I>::min() ||
668  n > std::numeric_limits<I>::max()) {
669  throw std::ios_base::failure("CompactSize exceeds limit of type");
670  }
671  v = n;
672  }
673 
674  template <typename Stream, typename I> void Ser(Stream &s, I v) {
675  static_assert(std::is_unsigned<I>::value,
676  "CompactSize only supported for unsigned integers");
677  static_assert(std::numeric_limits<I>::max() <=
678  std::numeric_limits<uint64_t>::max(),
679  "CompactSize only supports 64-bit integers and below");
680 
681  WriteCompactSize<Stream>(s, v);
682  }
683 };
684 
685 template <size_t Limit> struct LimitedStringFormatter {
686  template <typename Stream> void Unser(Stream &s, std::string &v) {
687  size_t size = ReadCompactSize(s);
688  if (size > Limit) {
689  throw std::ios_base::failure("String length limit exceeded");
690  }
691  v.resize(size);
692  if (size != 0) {
693  s.read((char *)v.data(), size);
694  }
695  }
696 
697  template <typename Stream> void Ser(Stream &s, const std::string &v) {
698  s << v;
699  }
700 };
701 
718 template <class Formatter> struct VectorFormatter {
719  template <typename Stream, typename V> void Ser(Stream &s, const V &v) {
720  Formatter formatter;
721  WriteCompactSize(s, v.size());
722  for (const typename V::value_type &elem : v) {
723  formatter.Ser(s, elem);
724  }
725  }
726 
727  template <typename Stream, typename V> void Unser(Stream &s, V &v) {
728  Formatter formatter;
729  v.clear();
730  size_t size = ReadCompactSize(s);
731  size_t allocated = 0;
732  while (allocated < size) {
733  // For DoS prevention, do not blindly allocate as much as the stream
734  // claims to contain. Instead, allocate in 5MiB batches, so that an
735  // attacker actually needs to provide X MiB of data to make us
736  // allocate X+5 Mib.
737  static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE,
738  "Vector element size too large");
739  allocated =
740  std::min(size, allocated + MAX_VECTOR_ALLOCATE /
741  sizeof(typename V::value_type));
742  v.reserve(allocated);
743  while (v.size() < allocated) {
744  v.emplace_back();
745  formatter.Unser(s, v.back());
746  }
747  }
748  };
749 };
750 
758 template <typename Stream, typename C>
759 void Serialize(Stream &os, const std::basic_string<C> &str);
760 template <typename Stream, typename C>
761 void Unserialize(Stream &is, std::basic_string<C> &str);
762 
768 template <typename Stream, unsigned int N, typename T>
769 void Serialize_impl(Stream &os, const prevector<N, T> &v, const uint8_t &);
770 template <typename Stream, unsigned int N, typename T, typename V>
771 void Serialize_impl(Stream &os, const prevector<N, T> &v, const V &);
772 template <typename Stream, unsigned int N, typename T>
773 inline void Serialize(Stream &os, const prevector<N, T> &v);
774 template <typename Stream, unsigned int N, typename T>
775 void Unserialize_impl(Stream &is, prevector<N, T> &v, const uint8_t &);
776 template <typename Stream, unsigned int N, typename T, typename V>
777 void Unserialize_impl(Stream &is, prevector<N, T> &v, const V &);
778 template <typename Stream, unsigned int N, typename T>
779 inline void Unserialize(Stream &is, prevector<N, T> &v);
780 
786 template <typename Stream, typename T, typename A>
787 void Serialize_impl(Stream &os, const std::vector<T, A> &v, const uint8_t &);
788 template <typename Stream, typename T, typename A>
789 void Serialize_impl(Stream &os, const std::vector<T, A> &v, const bool &);
790 template <typename Stream, typename T, typename A, typename V>
791 void Serialize_impl(Stream &os, const std::vector<T, A> &v, const V &);
792 template <typename Stream, typename T, typename A>
793 inline void Serialize(Stream &os, const std::vector<T, A> &v);
794 template <typename Stream, typename T, typename A>
795 void Unserialize_impl(Stream &is, std::vector<T, A> &v, const uint8_t &);
796 template <typename Stream, typename T, typename A, typename V>
797 void Unserialize_impl(Stream &is, std::vector<T, A> &v, const V &);
798 template <typename Stream, typename T, typename A>
799 inline void Unserialize(Stream &is, std::vector<T, A> &v);
800 
804 template <typename Stream, typename K, typename T>
805 void Serialize(Stream &os, const std::pair<K, T> &item);
806 template <typename Stream, typename K, typename T>
807 void Unserialize(Stream &is, std::pair<K, T> &item);
808 
812 template <typename Stream, typename K, typename T, typename Pred, typename A>
813 void Serialize(Stream &os, const std::map<K, T, Pred, A> &m);
814 template <typename Stream, typename K, typename T, typename Pred, typename A>
815 void Unserialize(Stream &is, std::map<K, T, Pred, A> &m);
816 
820 template <typename Stream, typename K, typename Pred, typename A>
821 void Serialize(Stream &os, const std::set<K, Pred, A> &m);
822 template <typename Stream, typename K, typename Pred, typename A>
823 void Unserialize(Stream &is, std::set<K, Pred, A> &m);
824 
828 template <typename Stream, typename T>
829 void Serialize(Stream &os, const std::shared_ptr<const T> &p);
830 template <typename Stream, typename T>
831 void Unserialize(Stream &os, std::shared_ptr<const T> &p);
832 
836 template <typename Stream, typename T>
837 void Serialize(Stream &os, const std::unique_ptr<const T> &p);
838 template <typename Stream, typename T>
839 void Unserialize(Stream &os, std::unique_ptr<const T> &p);
840 
845 template <typename Stream, typename T>
846 inline void Serialize(Stream &os, const T &a) {
847  a.Serialize(os);
848 }
849 
850 template <typename Stream, typename T>
851 inline void Unserialize(Stream &is, T &&a) {
852  a.Unserialize(is);
853 }
854 
862  template <typename Stream, typename T>
863  static void Ser(Stream &s, const T &t) {
864  Serialize(s, t);
865  }
866 
867  template <typename Stream, typename T> static void Unser(Stream &s, T &t) {
868  Unserialize(s, t);
869  }
870 };
871 
875 template <typename Stream, typename C>
876 void Serialize(Stream &os, const std::basic_string<C> &str) {
877  WriteCompactSize(os, str.size());
878  if (!str.empty()) {
879  os.write((char *)str.data(), str.size() * sizeof(C));
880  }
881 }
882 
883 template <typename Stream, typename C>
884 void Unserialize(Stream &is, std::basic_string<C> &str) {
885  size_t nSize = ReadCompactSize(is);
886  str.resize(nSize);
887  if (nSize != 0) {
888  is.read((char *)str.data(), nSize * sizeof(C));
889  }
890 }
891 
895 template <typename Stream, unsigned int N, typename T>
896 void Serialize_impl(Stream &os, const prevector<N, T> &v, const uint8_t &) {
897  WriteCompactSize(os, v.size());
898  if (!v.empty()) {
899  os.write((char *)v.data(), v.size() * sizeof(T));
900  }
901 }
902 
903 template <typename Stream, unsigned int N, typename T, typename V>
904 void Serialize_impl(Stream &os, const prevector<N, T> &v, const V &) {
906 }
907 
908 template <typename Stream, unsigned int N, typename T>
909 inline void Serialize(Stream &os, const prevector<N, T> &v) {
910  Serialize_impl(os, v, T());
911 }
912 
913 template <typename Stream, unsigned int N, typename T>
914 void Unserialize_impl(Stream &is, prevector<N, T> &v, const uint8_t &) {
915  // Limit size per read so bogus size value won't cause out of memory
916  v.clear();
917  size_t nSize = ReadCompactSize(is);
918  size_t i = 0;
919  while (i < nSize) {
920  size_t blk = std::min(nSize - i, size_t(1 + 4999999 / sizeof(T)));
921  v.resize_uninitialized(i + blk);
922  is.read((char *)&v[i], blk * sizeof(T));
923  i += blk;
924  }
925 }
926 
927 template <typename Stream, unsigned int N, typename T, typename V>
928 void Unserialize_impl(Stream &is, prevector<N, T> &v, const V &) {
930 }
931 
932 template <typename Stream, unsigned int N, typename T>
933 inline void Unserialize(Stream &is, prevector<N, T> &v) {
934  Unserialize_impl(is, v, T());
935 }
936 
940 template <typename Stream, typename T, typename A>
941 void Serialize_impl(Stream &os, const std::vector<T, A> &v, const uint8_t &) {
942  WriteCompactSize(os, v.size());
943  if (!v.empty()) {
944  os.write((char *)v.data(), v.size() * sizeof(T));
945  }
946 }
947 
948 template <typename Stream, typename T, typename A>
949 void Serialize_impl(Stream &os, const std::vector<T, A> &v, const bool &) {
950  // A special case for std::vector<bool>, as dereferencing
951  // std::vector<bool>::const_iterator does not result in a const bool&
952  // due to std::vector's special casing for bool arguments.
953  WriteCompactSize(os, v.size());
954  for (bool elem : v) {
955  ::Serialize(os, elem);
956  }
957 }
958 
959 template <typename Stream, typename T, typename A, typename V>
960 void Serialize_impl(Stream &os, const std::vector<T, A> &v, const V &) {
962 }
963 
964 template <typename Stream, typename T, typename A>
965 inline void Serialize(Stream &os, const std::vector<T, A> &v) {
966  Serialize_impl(os, v, T());
967 }
968 
969 template <typename Stream, typename T, typename A>
970 void Unserialize_impl(Stream &is, std::vector<T, A> &v, const uint8_t &) {
971  // Limit size per read so bogus size value won't cause out of memory
972  v.clear();
973  size_t nSize = ReadCompactSize(is);
974  size_t i = 0;
975  while (i < nSize) {
976  size_t blk = std::min(nSize - i, size_t(1 + 4999999 / sizeof(T)));
977  v.resize(i + blk);
978  is.read((char *)&v[i], blk * sizeof(T));
979  i += blk;
980  }
981 }
982 
983 template <typename Stream, typename T, typename A, typename V>
984 void Unserialize_impl(Stream &is, std::vector<T, A> &v, const V &) {
986 }
987 
988 template <typename Stream, typename T, typename A>
989 inline void Unserialize(Stream &is, std::vector<T, A> &v) {
990  Unserialize_impl(is, v, T());
991 }
992 
996 template <typename Stream, typename K, typename T>
997 void Serialize(Stream &os, const std::pair<K, T> &item) {
998  Serialize(os, item.first);
999  Serialize(os, item.second);
1000 }
1001 
1002 template <typename Stream, typename K, typename T>
1003 void Unserialize(Stream &is, std::pair<K, T> &item) {
1004  Unserialize(is, item.first);
1005  Unserialize(is, item.second);
1006 }
1007 
1011 template <typename Stream, typename K, typename T, typename Pred, typename A>
1012 void Serialize(Stream &os, const std::map<K, T, Pred, A> &m) {
1013  WriteCompactSize(os, m.size());
1014  for (const auto &entry : m) {
1015  Serialize(os, entry);
1016  }
1017 }
1018 
1019 template <typename Stream, typename K, typename T, typename Pred, typename A>
1020 void Unserialize(Stream &is, std::map<K, T, Pred, A> &m) {
1021  m.clear();
1022  size_t nSize = ReadCompactSize(is);
1023  typename std::map<K, T, Pred, A>::iterator mi = m.begin();
1024  for (size_t i = 0; i < nSize; i++) {
1025  std::pair<K, T> item;
1026  Unserialize(is, item);
1027  mi = m.insert(mi, item);
1028  }
1029 }
1030 
1034 template <typename Stream, typename K, typename Pred, typename A>
1035 void Serialize(Stream &os, const std::set<K, Pred, A> &m) {
1036  WriteCompactSize(os, m.size());
1037  for (const K &i : m) {
1038  Serialize(os, i);
1039  }
1040 }
1041 
1042 template <typename Stream, typename K, typename Pred, typename A>
1043 void Unserialize(Stream &is, std::set<K, Pred, A> &m) {
1044  m.clear();
1045  size_t nSize = ReadCompactSize(is);
1046  typename std::set<K, Pred, A>::iterator it = m.begin();
1047  for (size_t i = 0; i < nSize; i++) {
1048  K key;
1049  Unserialize(is, key);
1050  it = m.insert(it, key);
1051  }
1052 }
1053 
1057 template <typename Stream, typename T>
1058 void Serialize(Stream &os, const std::unique_ptr<const T> &p) {
1059  Serialize(os, *p);
1060 }
1061 
1062 template <typename Stream, typename T>
1063 void Unserialize(Stream &is, std::unique_ptr<const T> &p) {
1064  p.reset(new T(deserialize, is));
1065 }
1066 
1070 template <typename Stream, typename T>
1071 void Serialize(Stream &os, const std::shared_ptr<const T> &p) {
1072  Serialize(os, *p);
1073 }
1074 
1075 template <typename Stream, typename T>
1076 void Unserialize(Stream &is, std::shared_ptr<const T> &p) {
1077  p = std::make_shared<const T>(deserialize, is);
1078 }
1079 
1084  constexpr bool ForRead() const { return false; }
1085 };
1087  constexpr bool ForRead() const { return true; }
1088 };
1089 
1103 protected:
1104  size_t nSize;
1105 
1106  const int nVersion;
1107 
1108 public:
1109  explicit CSizeComputer(int nVersionIn) : nSize(0), nVersion(nVersionIn) {}
1110 
1111  void write(const char *psz, size_t _nSize) { this->nSize += _nSize; }
1112 
1114  void seek(size_t _nSize) { this->nSize += _nSize; }
1115 
1116  template <typename T> CSizeComputer &operator<<(const T &obj) {
1117  ::Serialize(*this, obj);
1118  return (*this);
1119  }
1120 
1121  size_t size() const { return nSize; }
1122 
1123  int GetVersion() const { return nVersion; }
1124 };
1125 
1126 template <typename Stream> void SerializeMany(Stream &s) {}
1127 
1128 template <typename Stream, typename Arg, typename... Args>
1129 void SerializeMany(Stream &s, const Arg &arg, const Args &... args) {
1130  ::Serialize(s, arg);
1131  ::SerializeMany(s, args...);
1132 }
1133 
1134 template <typename Stream> inline void UnserializeMany(Stream &s) {}
1135 
1136 template <typename Stream, typename Arg, typename... Args>
1137 inline void UnserializeMany(Stream &s, Arg &&arg, Args &&... args) {
1138  ::Unserialize(s, arg);
1139  ::UnserializeMany(s, args...);
1140 }
1141 
1142 template <typename Stream, typename... Args>
1143 inline void SerReadWriteMany(Stream &s, CSerActionSerialize ser_action,
1144  const Args &... args) {
1145  ::SerializeMany(s, args...);
1146 }
1147 
1148 template <typename Stream, typename... Args>
1149 inline void SerReadWriteMany(Stream &s, CSerActionUnserialize ser_action,
1150  Args &&... args) {
1151  ::UnserializeMany(s, args...);
1152 }
1153 
1154 template <typename Stream, typename Type, typename Fn>
1155 inline void SerRead(Stream &s, CSerActionSerialize ser_action, Type &&, Fn &&) {
1156 }
1157 
1158 template <typename Stream, typename Type, typename Fn>
1159 inline void SerRead(Stream &s, CSerActionUnserialize ser_action, Type &&obj,
1160  Fn &&fn) {
1161  fn(s, std::forward<Type>(obj));
1162 }
1163 
1164 template <typename Stream, typename Type, typename Fn>
1165 inline void SerWrite(Stream &s, CSerActionSerialize ser_action, Type &&obj,
1166  Fn &&fn) {
1167  fn(s, std::forward<Type>(obj));
1168 }
1169 
1170 template <typename Stream, typename Type, typename Fn>
1171 inline void SerWrite(Stream &s, CSerActionUnserialize ser_action, Type &&,
1172  Fn &&) {}
1173 
1174 template <typename I> inline void WriteVarInt(CSizeComputer &s, I n) {
1175  s.seek(GetSizeOfVarInt<I>(n));
1176 }
1177 
1178 inline void WriteCompactSize(CSizeComputer &s, uint64_t nSize) {
1179  s.seek(GetSizeOfCompactSize(nSize));
1180 }
1181 
1182 template <typename T> size_t GetSerializeSize(const T &t, int nVersion = 0) {
1183  return (CSizeComputer(nVersion) << t).size();
1184 }
1185 
1186 template <typename... T>
1187 size_t GetSerializeSizeMany(int nVersion, const T &... t) {
1188  CSizeComputer sc(nVersion);
1189  SerializeMany(sc, t...);
1190  return sc.size();
1191 }
1192 
1193 #endif // BITCOIN_SERIALIZE_H
DefaultFormatter
Default formatter.
Definition: serialize.h:861
GetSerializeSize
size_t GetSerializeSize(const T &t, int nVersion=0)
Definition: serialize.h:1182
htole16
uint16_t htole16(uint16_t host_16bits)
Definition: endian.h:150
deserialize_type
Dummy data type to identify deserializing constructors.
Definition: serialize.h:48
GetSizeOfVarInt
unsigned int GetSizeOfVarInt(I n)
Definition: serialize.h:504
Serialize_impl
void Serialize_impl(Stream &os, const prevector< N, T > &v, const uint8_t &)
prevector prevectors of uint8_t are a special case and are intended to be serialized as a single opaq...
Definition: serialize.h:896
ser_readdata32
uint32_t ser_readdata32(Stream &s)
Definition: serialize.h:112
ser_writedata16be
void ser_writedata16be(Stream &s, uint16_t obj)
Definition: serialize.h:78
CSizeComputer::GetVersion
int GetVersion() const
Definition: serialize.h:1123
DefaultFormatter::Unser
static void Unser(Stream &s, T &t)
Definition: serialize.h:867
ser_float_to_uint32
uint32_t ser_float_to_uint32(float x)
Definition: serialize.h:134
VarIntFormatter
Serialization wrapper class for integers in VarInt format.
Definition: serialize.h:603
htole64
uint64_t htole64(uint64_t host_64bits)
Definition: endian.h:198
DefaultFormatter::Ser
static void Ser(Stream &s, const T &t)
Definition: serialize.h:863
ser_writedata64
void ser_writedata64(Stream &s, uint64_t obj)
Definition: serialize.h:93
prevector::clear
void clear()
Definition: prevector.h:441
ReadVarInt
I ReadVarInt(Stream &is)
Definition: serialize.h:537
VectorFormatter
Formatter to serialize/deserialize vector elements using another formatter.
Definition: serialize.h:718
le32toh
uint32_t le32toh(uint32_t little_endian_32bits)
Definition: endian.h:186
GetSerializeSizeMany
size_t GetSerializeSizeMany(int nVersion, const T &... t)
Definition: serialize.h:1187
SerWrite
void SerWrite(Stream &s, CSerActionSerialize ser_action, Type &&obj, Fn &&fn)
Definition: serialize.h:1165
VarIntMode::NONNEGATIVE_SIGNED
@ NONNEGATIVE_SIGNED
ReadWriteAsHelper
X & ReadWriteAsHelper(X &x)
Convert the reference base type to X, without changing constness or reference type.
Definition: serialize.h:172
CSizeComputer::size
size_t size() const
Definition: serialize.h:1121
CSizeComputer::operator<<
CSizeComputer & operator<<(const T &obj)
Definition: serialize.h:1116
ReadCompactSize
uint64_t ReadCompactSize(Stream &is, bool range_check=true)
Decode a CompactSize-encoded variable-length integer.
Definition: serialize.h:430
CustomUintFormatter
Serialization wrapper class for custom integers and enums.
Definition: serialize.h:623
CheckVarIntMode
Definition: serialize.h:493
CSerActionSerialize
Support for SERIALIZE_METHODS and READWRITE macro.
Definition: serialize.h:1083
be64toh
uint64_t be64toh(uint64_t big_endian_64bits)
Definition: endian.h:204
CSerActionUnserialize
Definition: serialize.h:1086
be32toh
uint32_t be32toh(uint32_t big_endian_32bits)
Definition: endian.h:180
SerRead
void SerRead(Stream &s, CSerActionSerialize ser_action, Type &&, Fn &&)
Definition: serialize.h:1155
CustomUintFormatter::Unser
void Unser(Stream &s, I &v)
Definition: serialize.h:642
SER_NETWORK
@ SER_NETWORK
Definition: serialize.h:165
ser_uint64_to_double
double ser_uint64_to_double(uint64_t y)
Definition: serialize.h:141
CSizeComputer::nVersion
const int nVersion
Definition: serialize.h:1106
Span::size
constexpr std::size_t size() const noexcept
Definition: span.h:209
Span
A Span is an object that can refer to a contiguous sequence of objects.
Definition: span.h:93
CSerActionUnserialize::ForRead
constexpr bool ForRead() const
Definition: serialize.h:1087
ser_readdata32be
uint32_t ser_readdata32be(Stream &s)
Definition: serialize.h:117
MAX_SIZE
static constexpr uint64_t MAX_SIZE
The maximum size of a serialized object in bytes or number of elements (for eg vectors) when the size...
Definition: serialize.h:30
MAX_VECTOR_ALLOCATE
static const unsigned int MAX_VECTOR_ALLOCATE
Maximum amount of memory (in bytes) to allocate at once when deserializing vectors.
Definition: serialize.h:36
Unserialize
void Unserialize(Stream &s, char &a)
Definition: serialize.h:293
GetSizeOfCompactSize
uint32_t GetSizeOfCompactSize(uint64_t nSize)
Compact Size size < 253 – 1 byte size <= USHRT_MAX – 3 bytes (253 + 2 bytes) size <= UINT_MAX – 5 byt...
Definition: serialize.h:390
LimitedStringFormatter
Definition: serialize.h:685
ser_uint32_to_float
float ser_uint32_to_float(uint32_t y)
Definition: serialize.h:148
CSizeComputer
GetSerializeSize implementations
Definition: serialize.h:1102
LimitedStringFormatter::Ser
void Ser(Stream &s, const std::string &v)
Definition: serialize.h:697
ser_readdata16
uint16_t ser_readdata16(Stream &s)
Definition: serialize.h:102
WriteCompactSize
void WriteCompactSize(CSizeComputer &os, uint64_t nSize)
Definition: serialize.h:1178
span.h
le16toh
uint16_t le16toh(uint16_t little_endian_16bits)
Definition: endian.h:162
htobe32
uint32_t htobe32(uint32_t host_32bits)
Definition: endian.h:168
ser_readdata16be
uint16_t ser_readdata16be(Stream &s)
Definition: serialize.h:107
be16toh
uint16_t be16toh(uint16_t big_endian_16bits)
Definition: endian.h:156
Using
static Wrapper< Formatter, T & > Using(T &&t)
Cause serialization/deserialization of an object to be done using a specified formatter class.
Definition: serialize.h:591
CSerActionSerialize::ForRead
constexpr bool ForRead() const
Definition: serialize.h:1084
ser_readdata8
uint8_t ser_readdata8(Stream &s)
Definition: serialize.h:97
CompactSizeFormatter
Formatter for integers in CompactSize format.
Definition: serialize.h:664
htobe16
uint16_t htobe16(uint16_t host_16bits)
Definition: endian.h:144
VarIntMode
VarIntMode
Variable-length integers: bytes are a MSB base-128 encoding of the number.
Definition: serialize.h:491
CSizeComputer::CSizeComputer
CSizeComputer(int nVersionIn)
Definition: serialize.h:1109
UnserializeMany
void UnserializeMany(Stream &s)
Definition: serialize.h:1134
SerializeMany
void SerializeMany(Stream &s)
Definition: serialize.h:1126
CheckVarIntMode::CheckVarIntMode
constexpr CheckVarIntMode()
Definition: serialize.h:494
SerReadWriteMany
void SerReadWriteMany(Stream &s, CSerActionSerialize ser_action, const Args &... args)
Definition: serialize.h:1143
ser_writedata8
void ser_writedata8(Stream &s, uint8_t obj)
Lowest-level serialization and conversion.
Definition: serialize.h:69
CustomUintFormatter::Ser
void Ser(Stream &s, I v)
Definition: serialize.h:628
Wrapper::Unserialize
void Unserialize(Stream &s)
Definition: serialize.h:572
htole32
uint32_t htole32(uint32_t host_32bits)
Definition: endian.h:174
ser_writedata32
void ser_writedata32(Stream &s, uint32_t obj)
Definition: serialize.h:83
ser_double_to_uint64
uint64_t ser_double_to_uint64(double x)
Definition: serialize.h:127
CSizeComputer::write
void write(const char *psz, size_t _nSize)
Definition: serialize.h:1111
VarIntFormatter::Unser
void Unser(Stream &s, I &v)
Definition: serialize.h:608
CompactSizeFormatter::Ser
void Ser(Stream &s, I v)
Definition: serialize.h:674
LimitedStringFormatter::Unser
void Unser(Stream &s, std::string &v)
Definition: serialize.h:686
prevector
Implements a drop-in replacement for std::vector<T> which stores up to N elements directly (without h...
Definition: prevector.h:38
SER_GETHASH
@ SER_GETHASH
Definition: serialize.h:167
Span::data
constexpr C * data() const noexcept
Definition: span.h:198
le64toh
uint64_t le64toh(uint64_t little_endian_64bits)
Definition: endian.h:210
Wrapper::Serialize
void Serialize(Stream &s) const
Definition: serialize.h:569
WriteVarInt
void WriteVarInt(CSizeComputer &os, I n)
Definition: serialize.h:1174
prevector::data
value_type * data()
Definition: prevector.h:610
ser_writedata32be
void ser_writedata32be(Stream &s, uint32_t obj)
Definition: serialize.h:88
ser_writedata16
void ser_writedata16(Stream &s, uint16_t obj)
Definition: serialize.h:73
Wrapper::Wrapper
Wrapper(T obj)
Definition: serialize.h:568
prevector::size
size_type size() const
Definition: prevector.h:386
prevector::resize_uninitialized
void resize_uninitialized(size_type new_size)
Definition: prevector.h:482
VectorFormatter::Unser
void Unser(Stream &s, V &v)
Definition: serialize.h:727
SER_DISK
@ SER_DISK
Definition: serialize.h:166
prevector::empty
bool empty() const
Definition: prevector.h:388
Unserialize_impl
void Unserialize_impl(Stream &is, prevector< N, T > &v, const uint8_t &)
Definition: serialize.h:914
prevector.h
Wrapper::m_object
T m_object
Definition: serialize.h:562
Wrapper
Simple wrapper class to serialize objects using a formatter; used by Using().
Definition: serialize.h:560
VarIntMode::DEFAULT
@ DEFAULT
htobe64
uint64_t htobe64(uint64_t host_64bits)
Definition: endian.h:192
VectorFormatter::Ser
void Ser(Stream &s, const V &v)
Definition: serialize.h:719
CSizeComputer::seek
void seek(size_t _nSize)
Pretend _nSize bytes are written, without specifying them.
Definition: serialize.h:1114
Serialize
void Serialize(Stream &s, char a)
Definition: serialize.h:241
VarIntFormatter::Ser
void Ser(Stream &s, I v)
Definition: serialize.h:604
CharCast
char * CharCast(char *c)
Safely convert odd char pointer types to standard ones.
Definition: serialize.h:52
CompactSizeFormatter::Unser
void Unser(Stream &s, I &v)
Definition: serialize.h:665
deserialize
constexpr deserialize_type deserialize
Definition: serialize.h:49
ser_readdata64
uint64_t ser_readdata64(Stream &s)
Definition: serialize.h:122
CustomUintFormatter::MAX
static constexpr uint64_t MAX
Definition: serialize.h:626
CSizeComputer::nSize
size_t nSize
Definition: serialize.h:1104
endian.h