Bitcoin Core  24.99.0
P2P Digital Currency
serialize.h
Go to the documentation of this file.
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2021 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 
6 #ifndef BITCOIN_SERIALIZE_H
7 #define BITCOIN_SERIALIZE_H
8 
9 #include <compat/endian.h>
10 
11 #include <algorithm>
12 #include <cstdint>
13 #include <cstring>
14 #include <ios>
15 #include <limits>
16 #include <map>
17 #include <memory>
18 #include <set>
19 #include <string>
20 #include <string.h>
21 #include <utility>
22 #include <vector>
23 
24 #include <prevector.h>
25 #include <span.h>
26 
31 static constexpr uint64_t MAX_SIZE = 0x02000000;
32 
34 static const unsigned int MAX_VECTOR_ALLOCATE = 5000000;
35 
47 struct deserialize_type {};
49 
50 /*
51  * Lowest-level serialization and conversion.
52  */
53 template<typename Stream> inline void ser_writedata8(Stream &s, uint8_t obj)
54 {
55  s.write(AsBytes(Span{&obj, 1}));
56 }
57 template<typename Stream> inline void ser_writedata16(Stream &s, uint16_t obj)
58 {
59  obj = htole16(obj);
60  s.write(AsBytes(Span{&obj, 1}));
61 }
62 template<typename Stream> inline void ser_writedata16be(Stream &s, uint16_t obj)
63 {
64  obj = htobe16(obj);
65  s.write(AsBytes(Span{&obj, 1}));
66 }
67 template<typename Stream> inline void ser_writedata32(Stream &s, uint32_t obj)
68 {
69  obj = htole32(obj);
70  s.write(AsBytes(Span{&obj, 1}));
71 }
72 template<typename Stream> inline void ser_writedata32be(Stream &s, uint32_t obj)
73 {
74  obj = htobe32(obj);
75  s.write(AsBytes(Span{&obj, 1}));
76 }
77 template<typename Stream> inline void ser_writedata64(Stream &s, uint64_t obj)
78 {
79  obj = htole64(obj);
80  s.write(AsBytes(Span{&obj, 1}));
81 }
82 template<typename Stream> inline uint8_t ser_readdata8(Stream &s)
83 {
84  uint8_t obj;
85  s.read(AsWritableBytes(Span{&obj, 1}));
86  return obj;
87 }
88 template<typename Stream> inline uint16_t ser_readdata16(Stream &s)
89 {
90  uint16_t obj;
91  s.read(AsWritableBytes(Span{&obj, 1}));
92  return le16toh(obj);
93 }
94 template<typename Stream> inline uint16_t ser_readdata16be(Stream &s)
95 {
96  uint16_t obj;
97  s.read(AsWritableBytes(Span{&obj, 1}));
98  return be16toh(obj);
99 }
100 template<typename Stream> inline uint32_t ser_readdata32(Stream &s)
101 {
102  uint32_t obj;
103  s.read(AsWritableBytes(Span{&obj, 1}));
104  return le32toh(obj);
105 }
106 template<typename Stream> inline uint32_t ser_readdata32be(Stream &s)
107 {
108  uint32_t obj;
109  s.read(AsWritableBytes(Span{&obj, 1}));
110  return be32toh(obj);
111 }
112 template<typename Stream> inline uint64_t ser_readdata64(Stream &s)
113 {
114  uint64_t obj;
115  s.read(AsWritableBytes(Span{&obj, 1}));
116  return le64toh(obj);
117 }
118 
119 
121 //
122 // Templates for serializing to anything that looks like a stream,
123 // i.e. anything that supports .read(Span<std::byte>) and .write(Span<const std::byte>)
124 //
125 
126 class CSizeComputer;
127 
128 enum
129 {
130  // primary actions
131  SER_NETWORK = (1 << 0),
132  SER_DISK = (1 << 1),
133  SER_GETHASH = (1 << 2),
134 };
135 
137 template<typename X> X& ReadWriteAsHelper(X& x) { return x; }
138 template<typename X> const X& ReadWriteAsHelper(const X& x) { return x; }
139 
140 #define READWRITE(...) (::SerReadWriteMany(s, ser_action, __VA_ARGS__))
141 #define READWRITEAS(type, obj) (::SerReadWriteMany(s, ser_action, ReadWriteAsHelper<type>(obj)))
142 #define SER_READ(obj, code) ::SerRead(s, ser_action, obj, [&](Stream& s, typename std::remove_const<Type>::type& obj) { code; })
143 #define SER_WRITE(obj, code) ::SerWrite(s, ser_action, obj, [&](Stream& s, const Type& obj) { code; })
144 
161 #define FORMATTER_METHODS(cls, obj) \
162  template<typename Stream> \
163  static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, CSerActionSerialize()); } \
164  template<typename Stream> \
165  static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, CSerActionUnserialize()); } \
166  template<typename Stream, typename Type, typename Operation> \
167  static inline void SerializationOps(Type& obj, Stream& s, Operation ser_action) \
168 
176 #define SERIALIZE_METHODS(cls, obj) \
177  template<typename Stream> \
178  void Serialize(Stream& s) const \
179  { \
180  static_assert(std::is_same<const cls&, decltype(*this)>::value, "Serialize type mismatch"); \
181  Ser(s, *this); \
182  } \
183  template<typename Stream> \
184  void Unserialize(Stream& s) \
185  { \
186  static_assert(std::is_same<cls&, decltype(*this)>::value, "Unserialize type mismatch"); \
187  Unser(s, *this); \
188  } \
189  FORMATTER_METHODS(cls, obj)
190 
191 #ifndef CHAR_EQUALS_INT8
192 template <typename Stream> void Serialize(Stream&, char) = delete; // char serialization forbidden. Use uint8_t or int8_t
193 #endif
194 template<typename Stream> inline void Serialize(Stream& s, int8_t a ) { ser_writedata8(s, a); }
195 template<typename Stream> inline void Serialize(Stream& s, uint8_t a ) { ser_writedata8(s, a); }
196 template<typename Stream> inline void Serialize(Stream& s, int16_t a ) { ser_writedata16(s, a); }
197 template<typename Stream> inline void Serialize(Stream& s, uint16_t a) { ser_writedata16(s, a); }
198 template<typename Stream> inline void Serialize(Stream& s, int32_t a ) { ser_writedata32(s, a); }
199 template<typename Stream> inline void Serialize(Stream& s, uint32_t a) { ser_writedata32(s, a); }
200 template<typename Stream> inline void Serialize(Stream& s, int64_t a ) { ser_writedata64(s, a); }
201 template<typename Stream> inline void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); }
202 template<typename Stream, int N> inline void Serialize(Stream& s, const char (&a)[N]) { s.write(MakeByteSpan(a)); }
203 template<typename Stream, int N> inline void Serialize(Stream& s, const unsigned char (&a)[N]) { s.write(MakeByteSpan(a)); }
204 template<typename Stream> inline void Serialize(Stream& s, const Span<const unsigned char>& span) { s.write(AsBytes(span)); }
205 template<typename Stream> inline void Serialize(Stream& s, const Span<unsigned char>& span) { s.write(AsBytes(span)); }
206 
207 #ifndef CHAR_EQUALS_INT8
208 template <typename Stream> void Unserialize(Stream&, char) = delete; // char serialization forbidden. Use uint8_t or int8_t
209 #endif
210 template<typename Stream> inline void Unserialize(Stream& s, int8_t& a ) { a = ser_readdata8(s); }
211 template<typename Stream> inline void Unserialize(Stream& s, uint8_t& a ) { a = ser_readdata8(s); }
212 template<typename Stream> inline void Unserialize(Stream& s, int16_t& a ) { a = ser_readdata16(s); }
213 template<typename Stream> inline void Unserialize(Stream& s, uint16_t& a) { a = ser_readdata16(s); }
214 template<typename Stream> inline void Unserialize(Stream& s, int32_t& a ) { a = ser_readdata32(s); }
215 template<typename Stream> inline void Unserialize(Stream& s, uint32_t& a) { a = ser_readdata32(s); }
216 template<typename Stream> inline void Unserialize(Stream& s, int64_t& a ) { a = ser_readdata64(s); }
217 template<typename Stream> inline void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); }
218 template<typename Stream, int N> inline void Unserialize(Stream& s, char (&a)[N]) { s.read(MakeWritableByteSpan(a)); }
219 template<typename Stream, int N> inline void Unserialize(Stream& s, unsigned char (&a)[N]) { s.read(MakeWritableByteSpan(a)); }
220 template<typename Stream> inline void Unserialize(Stream& s, Span<unsigned char>& span) { s.read(AsWritableBytes(span)); }
221 
222 template <typename Stream> inline void Serialize(Stream& s, bool a) { uint8_t f = a; ser_writedata8(s, f); }
223 template <typename Stream> inline void Unserialize(Stream& s, bool& a) { uint8_t f = ser_readdata8(s); a = f; }
224 
225 
233 inline unsigned int GetSizeOfCompactSize(uint64_t nSize)
234 {
235  if (nSize < 253) return sizeof(unsigned char);
236  else if (nSize <= std::numeric_limits<uint16_t>::max()) return sizeof(unsigned char) + sizeof(uint16_t);
237  else if (nSize <= std::numeric_limits<unsigned int>::max()) return sizeof(unsigned char) + sizeof(unsigned int);
238  else return sizeof(unsigned char) + sizeof(uint64_t);
239 }
240 
241 inline void WriteCompactSize(CSizeComputer& os, uint64_t nSize);
242 
243 template<typename Stream>
244 void WriteCompactSize(Stream& os, uint64_t nSize)
245 {
246  if (nSize < 253)
247  {
248  ser_writedata8(os, nSize);
249  }
250  else if (nSize <= std::numeric_limits<uint16_t>::max())
251  {
252  ser_writedata8(os, 253);
253  ser_writedata16(os, nSize);
254  }
255  else if (nSize <= std::numeric_limits<unsigned int>::max())
256  {
257  ser_writedata8(os, 254);
258  ser_writedata32(os, nSize);
259  }
260  else
261  {
262  ser_writedata8(os, 255);
263  ser_writedata64(os, nSize);
264  }
265  return;
266 }
267 
274 template<typename Stream>
275 uint64_t ReadCompactSize(Stream& is, bool range_check = true)
276 {
277  uint8_t chSize = ser_readdata8(is);
278  uint64_t nSizeRet = 0;
279  if (chSize < 253)
280  {
281  nSizeRet = chSize;
282  }
283  else if (chSize == 253)
284  {
285  nSizeRet = ser_readdata16(is);
286  if (nSizeRet < 253)
287  throw std::ios_base::failure("non-canonical ReadCompactSize()");
288  }
289  else if (chSize == 254)
290  {
291  nSizeRet = ser_readdata32(is);
292  if (nSizeRet < 0x10000u)
293  throw std::ios_base::failure("non-canonical ReadCompactSize()");
294  }
295  else
296  {
297  nSizeRet = ser_readdata64(is);
298  if (nSizeRet < 0x100000000ULL)
299  throw std::ios_base::failure("non-canonical ReadCompactSize()");
300  }
301  if (range_check && nSizeRet > MAX_SIZE) {
302  throw std::ios_base::failure("ReadCompactSize(): size too large");
303  }
304  return nSizeRet;
305 }
306 
342 
343 template <VarIntMode Mode, typename I>
345  constexpr CheckVarIntMode()
346  {
347  static_assert(Mode != VarIntMode::DEFAULT || std::is_unsigned<I>::value, "Unsigned type required with mode DEFAULT.");
348  static_assert(Mode != VarIntMode::NONNEGATIVE_SIGNED || std::is_signed<I>::value, "Signed type required with mode NONNEGATIVE_SIGNED.");
349  }
350 };
351 
352 template<VarIntMode Mode, typename I>
353 inline unsigned int GetSizeOfVarInt(I n)
354 {
356  int nRet = 0;
357  while(true) {
358  nRet++;
359  if (n <= 0x7F)
360  break;
361  n = (n >> 7) - 1;
362  }
363  return nRet;
364 }
365 
366 template<typename I>
367 inline void WriteVarInt(CSizeComputer& os, I n);
368 
369 template<typename Stream, VarIntMode Mode, typename I>
370 void WriteVarInt(Stream& os, I n)
371 {
373  unsigned char tmp[(sizeof(n)*8+6)/7];
374  int len=0;
375  while(true) {
376  tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00);
377  if (n <= 0x7F)
378  break;
379  n = (n >> 7) - 1;
380  len++;
381  }
382  do {
383  ser_writedata8(os, tmp[len]);
384  } while(len--);
385 }
386 
387 template<typename Stream, VarIntMode Mode, typename I>
388 I ReadVarInt(Stream& is)
389 {
391  I n = 0;
392  while(true) {
393  unsigned char chData = ser_readdata8(is);
394  if (n > (std::numeric_limits<I>::max() >> 7)) {
395  throw std::ios_base::failure("ReadVarInt(): size too large");
396  }
397  n = (n << 7) | (chData & 0x7F);
398  if (chData & 0x80) {
399  if (n == std::numeric_limits<I>::max()) {
400  throw std::ios_base::failure("ReadVarInt(): size too large");
401  }
402  n++;
403  } else {
404  return n;
405  }
406  }
407 }
408 
410 template<typename Formatter, typename T>
411 class Wrapper
412 {
413  static_assert(std::is_lvalue_reference<T>::value, "Wrapper needs an lvalue reference type T");
414 protected:
416 public:
417  explicit Wrapper(T obj) : m_object(obj) {}
418  template<typename Stream> void Serialize(Stream &s) const { Formatter().Ser(s, m_object); }
419  template<typename Stream> void Unserialize(Stream &s) { Formatter().Unser(s, m_object); }
420 };
421 
432 template<typename Formatter, typename T>
433 static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T&>(t); }
434 
435 #define VARINT_MODE(obj, mode) Using<VarIntFormatter<mode>>(obj)
436 #define VARINT(obj) Using<VarIntFormatter<VarIntMode::DEFAULT>>(obj)
437 #define COMPACTSIZE(obj) Using<CompactSizeFormatter<true>>(obj)
438 #define LIMITED_STRING(obj,n) Using<LimitedStringFormatter<n>>(obj)
439 
441 template<VarIntMode Mode>
443 {
444  template<typename Stream, typename I> void Ser(Stream &s, I v)
445  {
446  WriteVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s, v);
447  }
448 
449  template<typename Stream, typename I> void Unser(Stream& s, I& v)
450  {
451  v = ReadVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s);
452  }
453 };
454 
464 template<int Bytes, bool BigEndian = false>
466 {
467  static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range");
468  static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes));
469 
470  template <typename Stream, typename I> void Ser(Stream& s, I v)
471  {
472  if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range");
473  if (BigEndian) {
474  uint64_t raw = htobe64(v);
475  s.write({AsBytePtr(&raw) + 8 - Bytes, Bytes});
476  } else {
477  uint64_t raw = htole64(v);
478  s.write({AsBytePtr(&raw), Bytes});
479  }
480  }
481 
482  template <typename Stream, typename I> void Unser(Stream& s, I& v)
483  {
484  using U = typename std::conditional<std::is_enum<I>::value, std::underlying_type<I>, std::common_type<I>>::type::type;
485  static_assert(std::numeric_limits<U>::max() >= MAX && std::numeric_limits<U>::min() <= 0, "Assigned type too small");
486  uint64_t raw = 0;
487  if (BigEndian) {
488  s.read({AsBytePtr(&raw) + 8 - Bytes, Bytes});
489  v = static_cast<I>(be64toh(raw));
490  } else {
491  s.read({AsBytePtr(&raw), Bytes});
492  v = static_cast<I>(le64toh(raw));
493  }
494  }
495 };
496 
498 
500 template<bool RangeCheck>
502 {
503  template<typename Stream, typename I>
504  void Unser(Stream& s, I& v)
505  {
506  uint64_t n = ReadCompactSize<Stream>(s, RangeCheck);
507  if (n < std::numeric_limits<I>::min() || n > std::numeric_limits<I>::max()) {
508  throw std::ios_base::failure("CompactSize exceeds limit of type");
509  }
510  v = n;
511  }
512 
513  template<typename Stream, typename I>
514  void Ser(Stream& s, I v)
515  {
516  static_assert(std::is_unsigned<I>::value, "CompactSize only supported for unsigned integers");
517  static_assert(std::numeric_limits<I>::max() <= std::numeric_limits<uint64_t>::max(), "CompactSize only supports 64-bit integers and below");
518 
519  WriteCompactSize<Stream>(s, v);
520  }
521 };
522 
523 template <typename U, bool LOSSY = false>
525  template <typename Stream, typename Tp>
526  void Unser(Stream& s, Tp& tp)
527  {
528  U u;
529  s >> u;
530  // Lossy deserialization does not make sense, so force Wnarrowing
531  tp = Tp{typename Tp::duration{typename Tp::duration::rep{u}}};
532  }
533  template <typename Stream, typename Tp>
534  void Ser(Stream& s, Tp tp)
535  {
536  if constexpr (LOSSY) {
537  s << U(tp.time_since_epoch().count());
538  } else {
539  s << U{tp.time_since_epoch().count()};
540  }
541  }
542 };
543 template <typename U>
545 
547 {
548 protected:
549  uint64_t n;
550 public:
551  explicit CompactSizeWriter(uint64_t n_in) : n(n_in) { }
552 
553  template<typename Stream>
554  void Serialize(Stream &s) const {
555  WriteCompactSize<Stream>(s, n);
556  }
557 };
558 
559 template<size_t Limit>
561 {
562  template<typename Stream>
563  void Unser(Stream& s, std::string& v)
564  {
565  size_t size = ReadCompactSize(s);
566  if (size > Limit) {
567  throw std::ios_base::failure("String length limit exceeded");
568  }
569  v.resize(size);
570  if (size != 0) s.read(MakeWritableByteSpan(v));
571  }
572 
573  template<typename Stream>
574  void Ser(Stream& s, const std::string& v)
575  {
576  s << v;
577  }
578 };
579 
593 template<class Formatter>
595 {
596  template<typename Stream, typename V>
597  void Ser(Stream& s, const V& v)
598  {
599  Formatter formatter;
600  WriteCompactSize(s, v.size());
601  for (const typename V::value_type& elem : v) {
602  formatter.Ser(s, elem);
603  }
604  }
605 
606  template<typename Stream, typename V>
607  void Unser(Stream& s, V& v)
608  {
609  Formatter formatter;
610  v.clear();
611  size_t size = ReadCompactSize(s);
612  size_t allocated = 0;
613  while (allocated < size) {
614  // For DoS prevention, do not blindly allocate as much as the stream claims to contain.
615  // Instead, allocate in 5MiB batches, so that an attacker actually needs to provide
616  // X MiB of data to make us allocate X+5 Mib.
617  static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE, "Vector element size too large");
618  allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type));
619  v.reserve(allocated);
620  while (v.size() < allocated) {
621  v.emplace_back();
622  formatter.Unser(s, v.back());
623  }
624  }
625  };
626 };
627 
635 template<typename Stream, typename C> void Serialize(Stream& os, const std::basic_string<C>& str);
636 template<typename Stream, typename C> void Unserialize(Stream& is, std::basic_string<C>& str);
637 
642 template<typename Stream, unsigned int N, typename T> void Serialize_impl(Stream& os, const prevector<N, T>& v, const unsigned char&);
643 template<typename Stream, unsigned int N, typename T, typename V> void Serialize_impl(Stream& os, const prevector<N, T>& v, const V&);
644 template<typename Stream, unsigned int N, typename T> inline void Serialize(Stream& os, const prevector<N, T>& v);
645 template<typename Stream, unsigned int N, typename T> void Unserialize_impl(Stream& is, prevector<N, T>& v, const unsigned char&);
646 template<typename Stream, unsigned int N, typename T, typename V> void Unserialize_impl(Stream& is, prevector<N, T>& v, const V&);
647 template<typename Stream, unsigned int N, typename T> inline void Unserialize(Stream& is, prevector<N, T>& v);
648 
653 template<typename Stream, typename T, typename A> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const unsigned char&);
654 template<typename Stream, typename T, typename A> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const bool&);
655 template<typename Stream, typename T, typename A, typename V> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&);
656 template<typename Stream, typename T, typename A> inline void Serialize(Stream& os, const std::vector<T, A>& v);
657 template<typename Stream, typename T, typename A> void Unserialize_impl(Stream& is, std::vector<T, A>& v, const unsigned char&);
658 template<typename Stream, typename T, typename A, typename V> void Unserialize_impl(Stream& is, std::vector<T, A>& v, const V&);
659 template<typename Stream, typename T, typename A> inline void Unserialize(Stream& is, std::vector<T, A>& v);
660 
664 template<typename Stream, typename K, typename T> void Serialize(Stream& os, const std::pair<K, T>& item);
665 template<typename Stream, typename K, typename T> void Unserialize(Stream& is, std::pair<K, T>& item);
666 
670 template<typename Stream, typename K, typename T, typename Pred, typename A> void Serialize(Stream& os, const std::map<K, T, Pred, A>& m);
671 template<typename Stream, typename K, typename T, typename Pred, typename A> void Unserialize(Stream& is, std::map<K, T, Pred, A>& m);
672 
676 template<typename Stream, typename K, typename Pred, typename A> void Serialize(Stream& os, const std::set<K, Pred, A>& m);
677 template<typename Stream, typename K, typename Pred, typename A> void Unserialize(Stream& is, std::set<K, Pred, A>& m);
678 
682 template<typename Stream, typename T> void Serialize(Stream& os, const std::shared_ptr<const T>& p);
683 template<typename Stream, typename T> void Unserialize(Stream& os, std::shared_ptr<const T>& p);
684 
688 template<typename Stream, typename T> void Serialize(Stream& os, const std::unique_ptr<const T>& p);
689 template<typename Stream, typename T> void Unserialize(Stream& os, std::unique_ptr<const T>& p);
690 
691 
692 
696 template<typename Stream, typename T>
697 inline void Serialize(Stream& os, const T& a)
698 {
699  a.Serialize(os);
700 }
701 
702 template<typename Stream, typename T>
703 inline void Unserialize(Stream& is, T&& a)
704 {
705  a.Unserialize(is);
706 }
707 
714 {
715  template<typename Stream, typename T>
716  static void Ser(Stream& s, const T& t) { Serialize(s, t); }
717 
718  template<typename Stream, typename T>
719  static void Unser(Stream& s, T& t) { Unserialize(s, t); }
720 };
721 
722 
723 
724 
725 
729 template<typename Stream, typename C>
730 void Serialize(Stream& os, const std::basic_string<C>& str)
731 {
732  WriteCompactSize(os, str.size());
733  if (!str.empty())
734  os.write(MakeByteSpan(str));
735 }
736 
737 template<typename Stream, typename C>
738 void Unserialize(Stream& is, std::basic_string<C>& str)
739 {
740  unsigned int nSize = ReadCompactSize(is);
741  str.resize(nSize);
742  if (nSize != 0)
743  is.read(MakeWritableByteSpan(str));
744 }
745 
746 
747 
751 template<typename Stream, unsigned int N, typename T>
752 void Serialize_impl(Stream& os, const prevector<N, T>& v, const unsigned char&)
753 {
754  WriteCompactSize(os, v.size());
755  if (!v.empty())
756  os.write(MakeByteSpan(v));
757 }
758 
759 template<typename Stream, unsigned int N, typename T, typename V>
760 void Serialize_impl(Stream& os, const prevector<N, T>& v, const V&)
761 {
763 }
764 
765 template<typename Stream, unsigned int N, typename T>
766 inline void Serialize(Stream& os, const prevector<N, T>& v)
767 {
768  Serialize_impl(os, v, T());
769 }
770 
771 
772 template<typename Stream, unsigned int N, typename T>
773 void Unserialize_impl(Stream& is, prevector<N, T>& v, const unsigned char&)
774 {
775  // Limit size per read so bogus size value won't cause out of memory
776  v.clear();
777  unsigned int nSize = ReadCompactSize(is);
778  unsigned int i = 0;
779  while (i < nSize)
780  {
781  unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T)));
782  v.resize_uninitialized(i + blk);
783  is.read(AsWritableBytes(Span{&v[i], blk}));
784  i += blk;
785  }
786 }
787 
788 template<typename Stream, unsigned int N, typename T, typename V>
789 void Unserialize_impl(Stream& is, prevector<N, T>& v, const V&)
790 {
792 }
793 
794 template<typename Stream, unsigned int N, typename T>
795 inline void Unserialize(Stream& is, prevector<N, T>& v)
796 {
797  Unserialize_impl(is, v, T());
798 }
799 
800 
801 
805 template<typename Stream, typename T, typename A>
806 void Serialize_impl(Stream& os, const std::vector<T, A>& v, const unsigned char&)
807 {
808  WriteCompactSize(os, v.size());
809  if (!v.empty())
810  os.write(MakeByteSpan(v));
811 }
812 
813 template<typename Stream, typename T, typename A>
814 void Serialize_impl(Stream& os, const std::vector<T, A>& v, const bool&)
815 {
816  // A special case for std::vector<bool>, as dereferencing
817  // std::vector<bool>::const_iterator does not result in a const bool&
818  // due to std::vector's special casing for bool arguments.
819  WriteCompactSize(os, v.size());
820  for (bool elem : v) {
821  ::Serialize(os, elem);
822  }
823 }
824 
825 template<typename Stream, typename T, typename A, typename V>
826 void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&)
827 {
829 }
830 
831 template<typename Stream, typename T, typename A>
832 inline void Serialize(Stream& os, const std::vector<T, A>& v)
833 {
834  Serialize_impl(os, v, T());
835 }
836 
837 
838 template<typename Stream, typename T, typename A>
839 void Unserialize_impl(Stream& is, std::vector<T, A>& v, const unsigned char&)
840 {
841  // Limit size per read so bogus size value won't cause out of memory
842  v.clear();
843  unsigned int nSize = ReadCompactSize(is);
844  unsigned int i = 0;
845  while (i < nSize)
846  {
847  unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T)));
848  v.resize(i + blk);
849  is.read(AsWritableBytes(Span{&v[i], blk}));
850  i += blk;
851  }
852 }
853 
854 template<typename Stream, typename T, typename A, typename V>
855 void Unserialize_impl(Stream& is, std::vector<T, A>& v, const V&)
856 {
858 }
859 
860 template<typename Stream, typename T, typename A>
861 inline void Unserialize(Stream& is, std::vector<T, A>& v)
862 {
863  Unserialize_impl(is, v, T());
864 }
865 
866 
867 
871 template<typename Stream, typename K, typename T>
872 void Serialize(Stream& os, const std::pair<K, T>& item)
873 {
874  Serialize(os, item.first);
875  Serialize(os, item.second);
876 }
877 
878 template<typename Stream, typename K, typename T>
879 void Unserialize(Stream& is, std::pair<K, T>& item)
880 {
881  Unserialize(is, item.first);
882  Unserialize(is, item.second);
883 }
884 
885 
886 
890 template<typename Stream, typename K, typename T, typename Pred, typename A>
891 void Serialize(Stream& os, const std::map<K, T, Pred, A>& m)
892 {
893  WriteCompactSize(os, m.size());
894  for (const auto& entry : m)
895  Serialize(os, entry);
896 }
897 
898 template<typename Stream, typename K, typename T, typename Pred, typename A>
899 void Unserialize(Stream& is, std::map<K, T, Pred, A>& m)
900 {
901  m.clear();
902  unsigned int nSize = ReadCompactSize(is);
903  typename std::map<K, T, Pred, A>::iterator mi = m.begin();
904  for (unsigned int i = 0; i < nSize; i++)
905  {
906  std::pair<K, T> item;
907  Unserialize(is, item);
908  mi = m.insert(mi, item);
909  }
910 }
911 
912 
913 
917 template<typename Stream, typename K, typename Pred, typename A>
918 void Serialize(Stream& os, const std::set<K, Pred, A>& m)
919 {
920  WriteCompactSize(os, m.size());
921  for (typename std::set<K, Pred, A>::const_iterator it = m.begin(); it != m.end(); ++it)
922  Serialize(os, (*it));
923 }
924 
925 template<typename Stream, typename K, typename Pred, typename A>
926 void Unserialize(Stream& is, std::set<K, Pred, A>& m)
927 {
928  m.clear();
929  unsigned int nSize = ReadCompactSize(is);
930  typename std::set<K, Pred, A>::iterator it = m.begin();
931  for (unsigned int i = 0; i < nSize; i++)
932  {
933  K key;
934  Unserialize(is, key);
935  it = m.insert(it, key);
936  }
937 }
938 
939 
940 
944 template<typename Stream, typename T> void
945 Serialize(Stream& os, const std::unique_ptr<const T>& p)
946 {
947  Serialize(os, *p);
948 }
949 
950 template<typename Stream, typename T>
951 void Unserialize(Stream& is, std::unique_ptr<const T>& p)
952 {
953  p.reset(new T(deserialize, is));
954 }
955 
956 
957 
961 template<typename Stream, typename T> void
962 Serialize(Stream& os, const std::shared_ptr<const T>& p)
963 {
964  Serialize(os, *p);
965 }
966 
967 template<typename Stream, typename T>
968 void Unserialize(Stream& is, std::shared_ptr<const T>& p)
969 {
970  p = std::make_shared<const T>(deserialize, is);
971 }
972 
973 
974 
979 {
980  constexpr bool ForRead() const { return false; }
981 };
983 {
984  constexpr bool ForRead() const { return true; }
985 };
986 
987 
988 
989 
990 
991 
992 
993 
994 /* ::GetSerializeSize implementations
995  *
996  * Computing the serialized size of objects is done through a special stream
997  * object of type CSizeComputer, which only records the number of bytes written
998  * to it.
999  *
1000  * If your Serialize or SerializationOp method has non-trivial overhead for
1001  * serialization, it may be worthwhile to implement a specialized version for
1002  * CSizeComputer, which uses the s.seek() method to record bytes that would
1003  * be written instead.
1004  */
1006 {
1007 protected:
1008  size_t nSize;
1009 
1010  const int nVersion;
1011 public:
1012  explicit CSizeComputer(int nVersionIn) : nSize(0), nVersion(nVersionIn) {}
1013 
1015  {
1016  this->nSize += src.size();
1017  }
1018 
1020  void seek(size_t _nSize)
1021  {
1022  this->nSize += _nSize;
1023  }
1024 
1025  template<typename T>
1027  {
1028  ::Serialize(*this, obj);
1029  return (*this);
1030  }
1031 
1032  size_t size() const {
1033  return nSize;
1034  }
1035 
1036  int GetVersion() const { return nVersion; }
1037 };
1038 
1039 template<typename Stream>
1040 void SerializeMany(Stream& s)
1041 {
1042 }
1043 
1044 template<typename Stream, typename Arg, typename... Args>
1045 void SerializeMany(Stream& s, const Arg& arg, const Args&... args)
1046 {
1047  ::Serialize(s, arg);
1048  ::SerializeMany(s, args...);
1049 }
1050 
1051 template<typename Stream>
1052 inline void UnserializeMany(Stream& s)
1053 {
1054 }
1055 
1056 template<typename Stream, typename Arg, typename... Args>
1057 inline void UnserializeMany(Stream& s, Arg&& arg, Args&&... args)
1058 {
1059  ::Unserialize(s, arg);
1060  ::UnserializeMany(s, args...);
1061 }
1062 
1063 template<typename Stream, typename... Args>
1064 inline void SerReadWriteMany(Stream& s, CSerActionSerialize ser_action, const Args&... args)
1065 {
1066  ::SerializeMany(s, args...);
1067 }
1068 
1069 template<typename Stream, typename... Args>
1070 inline void SerReadWriteMany(Stream& s, CSerActionUnserialize ser_action, Args&&... args)
1071 {
1072  ::UnserializeMany(s, args...);
1073 }
1074 
1075 template<typename Stream, typename Type, typename Fn>
1076 inline void SerRead(Stream& s, CSerActionSerialize ser_action, Type&&, Fn&&)
1077 {
1078 }
1079 
1080 template<typename Stream, typename Type, typename Fn>
1081 inline void SerRead(Stream& s, CSerActionUnserialize ser_action, Type&& obj, Fn&& fn)
1082 {
1083  fn(s, std::forward<Type>(obj));
1084 }
1085 
1086 template<typename Stream, typename Type, typename Fn>
1087 inline void SerWrite(Stream& s, CSerActionSerialize ser_action, Type&& obj, Fn&& fn)
1088 {
1089  fn(s, std::forward<Type>(obj));
1090 }
1091 
1092 template<typename Stream, typename Type, typename Fn>
1093 inline void SerWrite(Stream& s, CSerActionUnserialize ser_action, Type&&, Fn&&)
1094 {
1095 }
1096 
1097 template<typename I>
1098 inline void WriteVarInt(CSizeComputer &s, I n)
1099 {
1100  s.seek(GetSizeOfVarInt<I>(n));
1101 }
1102 
1103 inline void WriteCompactSize(CSizeComputer &s, uint64_t nSize)
1104 {
1105  s.seek(GetSizeOfCompactSize(nSize));
1106 }
1107 
1108 template <typename T>
1109 size_t GetSerializeSize(const T& t, int nVersion = 0)
1110 {
1111  return (CSizeComputer(nVersion) << t).size();
1112 }
1113 
1114 template <typename... T>
1115 size_t GetSerializeSizeMany(int nVersion, const T&... t)
1116 {
1117  CSizeComputer sc(nVersion);
1118  SerializeMany(sc, t...);
1119  return sc.size();
1120 }
1121 
1122 #endif // BITCOIN_SERIALIZE_H
CSizeComputer & operator<<(const T &obj)
Definition: serialize.h:1026
void write(Span< const std::byte > src)
Definition: serialize.h:1014
CSizeComputer(int nVersionIn)
Definition: serialize.h:1012
size_t nSize
Definition: serialize.h:1008
size_t size() const
Definition: serialize.h:1032
const int nVersion
Definition: serialize.h:1010
void seek(size_t _nSize)
Pretend _nSize bytes are written, without specifying them.
Definition: serialize.h:1020
int GetVersion() const
Definition: serialize.h:1036
void Serialize(Stream &s) const
Definition: serialize.h:554
CompactSizeWriter(uint64_t n_in)
Definition: serialize.h:551
A Span is an object that can refer to a contiguous sequence of objects.
Definition: span.h:97
constexpr std::size_t size() const noexcept
Definition: span.h:186
Simple wrapper class to serialize objects using a formatter; used by Using().
Definition: serialize.h:412
Wrapper(T obj)
Definition: serialize.h:417
T m_object
Definition: serialize.h:413
void Serialize(Stream &s) const
Definition: serialize.h:418
void Unserialize(Stream &s)
Definition: serialize.h:419
Implements a drop-in replacement for std::vector<T> which stores up to N elements directly (without h...
Definition: prevector.h:37
bool empty() const
Definition: prevector.h:288
void clear()
Definition: prevector.h:345
size_type size() const
Definition: prevector.h:284
void resize_uninitialized(size_type new_size)
Definition: prevector.h:388
uint16_t be16toh(uint16_t big_endian_16bits)
Definition: endian.h:170
uint16_t htobe16(uint16_t host_16bits)
Definition: endian.h:156
uint32_t le32toh(uint32_t little_endian_32bits)
Definition: endian.h:205
uint32_t htobe32(uint32_t host_32bits)
Definition: endian.h:184
uint16_t le16toh(uint16_t little_endian_16bits)
Definition: endian.h:177
uint64_t htobe64(uint64_t host_64bits)
Definition: endian.h:212
uint64_t be64toh(uint64_t big_endian_64bits)
Definition: endian.h:226
uint32_t be32toh(uint32_t big_endian_32bits)
Definition: endian.h:198
uint64_t htole64(uint64_t host_64bits)
Definition: endian.h:219
uint32_t htole32(uint32_t host_32bits)
Definition: endian.h:191
uint16_t htole16(uint16_t host_16bits)
Definition: endian.h:163
uint64_t le64toh(uint64_t little_endian_64bits)
Definition: endian.h:233
#define T(expected, seed, data)
#define X(name)
Definition: net.cpp:607
ArgsManager args
void SerializeMany(Stream &s)
Definition: serialize.h:1040
@ SER_DISK
Definition: serialize.h:132
@ SER_NETWORK
Definition: serialize.h:131
@ SER_GETHASH
Definition: serialize.h:133
static const unsigned int MAX_VECTOR_ALLOCATE
Maximum amount of memory (in bytes) to allocate at once when deserializing vectors.
Definition: serialize.h:34
uint8_t ser_readdata8(Stream &s)
Definition: serialize.h:82
I ReadVarInt(Stream &is)
Definition: serialize.h:388
void ser_writedata32be(Stream &s, uint32_t obj)
Definition: serialize.h:72
void WriteVarInt(CSizeComputer &os, I n)
Definition: serialize.h:1098
VarIntMode
Variable-length integers: bytes are a MSB base-128 encoding of the number.
Definition: serialize.h:341
@ NONNEGATIVE_SIGNED
void ser_writedata32(Stream &s, uint32_t obj)
Definition: serialize.h:67
static constexpr uint64_t MAX_SIZE
The maximum size of a serialized object in bytes or number of elements (for eg vectors) when the size...
Definition: serialize.h:31
unsigned int GetSizeOfCompactSize(uint64_t nSize)
Compact Size size < 253 – 1 byte size <= USHRT_MAX – 3 bytes (253 + 2 bytes) size <= UINT_MAX – 5 byt...
Definition: serialize.h:233
constexpr deserialize_type deserialize
Definition: serialize.h:48
void ser_writedata16(Stream &s, uint16_t obj)
Definition: serialize.h:57
void Serialize_impl(Stream &os, const prevector< N, T > &v, const unsigned char &)
prevector prevectors of unsigned char are a special case and are intended to be serialized as a singl...
Definition: serialize.h:752
X & ReadWriteAsHelper(X &x)
Convert the reference base type to X, without changing constness or reference type.
Definition: serialize.h:137
void SerReadWriteMany(Stream &s, CSerActionSerialize ser_action, const Args &... args)
Definition: serialize.h:1064
void Unserialize(Stream &, char)=delete
static Wrapper< Formatter, T & > Using(T &&t)
Cause serialization/deserialization of an object to be done using a specified formatter class.
Definition: serialize.h:433
void Unserialize_impl(Stream &is, prevector< N, T > &v, const unsigned char &)
Definition: serialize.h:773
void SerWrite(Stream &s, CSerActionSerialize ser_action, Type &&obj, Fn &&fn)
Definition: serialize.h:1087
void ser_writedata16be(Stream &s, uint16_t obj)
Definition: serialize.h:62
uint16_t ser_readdata16(Stream &s)
Definition: serialize.h:88
uint64_t ser_readdata64(Stream &s)
Definition: serialize.h:112
void ser_writedata8(Stream &s, uint8_t obj)
Definition: serialize.h:53
uint64_t ReadCompactSize(Stream &is, bool range_check=true)
Decode a CompactSize-encoded variable-length integer.
Definition: serialize.h:275
unsigned int GetSizeOfVarInt(I n)
Definition: serialize.h:353
uint32_t ser_readdata32(Stream &s)
Definition: serialize.h:100
uint16_t ser_readdata16be(Stream &s)
Definition: serialize.h:94
void SerRead(Stream &s, CSerActionSerialize ser_action, Type &&, Fn &&)
Definition: serialize.h:1076
size_t GetSerializeSize(const T &t, int nVersion=0)
Definition: serialize.h:1109
void UnserializeMany(Stream &s)
Definition: serialize.h:1052
size_t GetSerializeSizeMany(int nVersion, const T &... t)
Definition: serialize.h:1115
void ser_writedata64(Stream &s, uint64_t obj)
Definition: serialize.h:77
uint32_t ser_readdata32be(Stream &s)
Definition: serialize.h:106
void Serialize(Stream &, char)=delete
void WriteCompactSize(CSizeComputer &os, uint64_t nSize)
Definition: serialize.h:1103
Span< std::byte > AsWritableBytes(Span< T > s) noexcept
Definition: span.h:258
Span< const std::byte > MakeByteSpan(V &&v) noexcept
Definition: span.h:264
Span< const std::byte > AsBytes(Span< T > s) noexcept
Definition: span.h:253
const std::byte * AsBytePtr(const void *data)
Convert a data pointer to a std::byte data pointer.
Definition: span.h:248
Span< std::byte > MakeWritableByteSpan(V &&v) noexcept
Definition: span.h:269
Support for SERIALIZE_METHODS and READWRITE macro.
Definition: serialize.h:979
constexpr bool ForRead() const
Definition: serialize.h:980
constexpr bool ForRead() const
Definition: serialize.h:984
constexpr CheckVarIntMode()
Definition: serialize.h:345
void Unser(Stream &s, Tp &tp)
Definition: serialize.h:526
void Ser(Stream &s, Tp tp)
Definition: serialize.h:534
Formatter for integers in CompactSize format.
Definition: serialize.h:502
void Unser(Stream &s, I &v)
Definition: serialize.h:504
void Ser(Stream &s, I v)
Definition: serialize.h:514
Serialization wrapper class for custom integers and enums.
Definition: serialize.h:466
static constexpr uint64_t MAX
Definition: serialize.h:468
void Ser(Stream &s, I v)
Definition: serialize.h:470
void Unser(Stream &s, I &v)
Definition: serialize.h:482
Default formatter.
Definition: serialize.h:714
static void Ser(Stream &s, const T &t)
Definition: serialize.h:716
static void Unser(Stream &s, T &t)
Definition: serialize.h:719
void Unser(Stream &s, std::string &v)
Definition: serialize.h:563
void Ser(Stream &s, const std::string &v)
Definition: serialize.h:574
Serialization wrapper class for integers in VarInt format.
Definition: serialize.h:443
void Ser(Stream &s, I v)
Definition: serialize.h:444
void Unser(Stream &s, I &v)
Definition: serialize.h:449
Formatter to serialize/deserialize vector elements using another formatter.
Definition: serialize.h:595
void Unser(Stream &s, V &v)
Definition: serialize.h:607
void Ser(Stream &s, const V &v)
Definition: serialize.h:597
Dummy data type to identify deserializing constructors.
Definition: serialize.h:47