Bitcoin Core  27.99.0
P2P Digital Currency
serialize.h
Go to the documentation of this file.
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2022 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 
6 #ifndef BITCOIN_SERIALIZE_H
7 #define BITCOIN_SERIALIZE_H
8 
9 #include <attributes.h>
10 #include <compat/assumptions.h> // IWYU pragma: keep
11 #include <compat/endian.h>
12 #include <prevector.h>
13 #include <span.h>
14 
15 #include <algorithm>
16 #include <concepts>
17 #include <cstdint>
18 #include <cstring>
19 #include <ios>
20 #include <limits>
21 #include <map>
22 #include <memory>
23 #include <set>
24 #include <string>
25 #include <utility>
26 #include <vector>
27 
32 static constexpr uint64_t MAX_SIZE = 0x02000000;
33 
35 static const unsigned int MAX_VECTOR_ALLOCATE = 5000000;
36 
48 struct deserialize_type {};
50 
51 /*
52  * Lowest-level serialization and conversion.
53  */
54 template<typename Stream> inline void ser_writedata8(Stream &s, uint8_t obj)
55 {
56  s.write(AsBytes(Span{&obj, 1}));
57 }
58 template<typename Stream> inline void ser_writedata16(Stream &s, uint16_t obj)
59 {
60  obj = htole16_internal(obj);
61  s.write(AsBytes(Span{&obj, 1}));
62 }
63 template<typename Stream> inline void ser_writedata16be(Stream &s, uint16_t obj)
64 {
65  obj = htobe16_internal(obj);
66  s.write(AsBytes(Span{&obj, 1}));
67 }
68 template<typename Stream> inline void ser_writedata32(Stream &s, uint32_t obj)
69 {
70  obj = htole32_internal(obj);
71  s.write(AsBytes(Span{&obj, 1}));
72 }
73 template<typename Stream> inline void ser_writedata32be(Stream &s, uint32_t obj)
74 {
75  obj = htobe32_internal(obj);
76  s.write(AsBytes(Span{&obj, 1}));
77 }
78 template<typename Stream> inline void ser_writedata64(Stream &s, uint64_t obj)
79 {
80  obj = htole64_internal(obj);
81  s.write(AsBytes(Span{&obj, 1}));
82 }
83 template<typename Stream> inline uint8_t ser_readdata8(Stream &s)
84 {
85  uint8_t obj;
86  s.read(AsWritableBytes(Span{&obj, 1}));
87  return obj;
88 }
89 template<typename Stream> inline uint16_t ser_readdata16(Stream &s)
90 {
91  uint16_t obj;
92  s.read(AsWritableBytes(Span{&obj, 1}));
93  return le16toh_internal(obj);
94 }
95 template<typename Stream> inline uint16_t ser_readdata16be(Stream &s)
96 {
97  uint16_t obj;
98  s.read(AsWritableBytes(Span{&obj, 1}));
99  return be16toh_internal(obj);
100 }
101 template<typename Stream> inline uint32_t ser_readdata32(Stream &s)
102 {
103  uint32_t obj;
104  s.read(AsWritableBytes(Span{&obj, 1}));
105  return le32toh_internal(obj);
106 }
107 template<typename Stream> inline uint32_t ser_readdata32be(Stream &s)
108 {
109  uint32_t obj;
110  s.read(AsWritableBytes(Span{&obj, 1}));
111  return be32toh_internal(obj);
112 }
113 template<typename Stream> inline uint64_t ser_readdata64(Stream &s)
114 {
115  uint64_t obj;
116  s.read(AsWritableBytes(Span{&obj, 1}));
117  return le64toh_internal(obj);
118 }
119 
120 
121 class SizeComputer;
122 
143 template <class Out, class In>
145 {
146  static_assert(std::is_base_of_v<Out, In>);
147  return x;
148 }
149 template <class Out, class In>
150 const Out& AsBase(const In& x)
151 {
152  static_assert(std::is_base_of_v<Out, In>);
153  return x;
154 }
155 
156 #define READWRITE(...) (ser_action.SerReadWriteMany(s, __VA_ARGS__))
157 #define SER_READ(obj, code) ser_action.SerRead(s, obj, [&](Stream& s, typename std::remove_const<Type>::type& obj) { code; })
158 #define SER_WRITE(obj, code) ser_action.SerWrite(s, obj, [&](Stream& s, const Type& obj) { code; })
159 
176 #define FORMATTER_METHODS(cls, obj) \
177  template<typename Stream> \
178  static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, ActionSerialize{}); } \
179  template<typename Stream> \
180  static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, ActionUnserialize{}); } \
181  template<typename Stream, typename Type, typename Operation> \
182  static void SerializationOps(Type& obj, Stream& s, Operation ser_action)
183 
217 #define FORMATTER_METHODS_PARAMS(cls, obj, paramcls, paramobj) \
218  template <typename Stream> \
219  static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, ActionSerialize{}, s.GetParams()); } \
220  template <typename Stream> \
221  static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, ActionUnserialize{}, s.GetParams()); } \
222  template <typename Stream, typename Type, typename Operation> \
223  static void SerializationOps(Type& obj, Stream& s, Operation ser_action, const paramcls& paramobj)
224 
225 #define BASE_SERIALIZE_METHODS(cls) \
226  template <typename Stream> \
227  void Serialize(Stream& s) const \
228  { \
229  static_assert(std::is_same<const cls&, decltype(*this)>::value, "Serialize type mismatch"); \
230  Ser(s, *this); \
231  } \
232  template <typename Stream> \
233  void Unserialize(Stream& s) \
234  { \
235  static_assert(std::is_same<cls&, decltype(*this)>::value, "Unserialize type mismatch"); \
236  Unser(s, *this); \
237  }
238 
246 #define SERIALIZE_METHODS(cls, obj) \
247  BASE_SERIALIZE_METHODS(cls) \
248  FORMATTER_METHODS(cls, obj)
249 
255 #define SERIALIZE_METHODS_PARAMS(cls, obj, paramcls, paramobj) \
256  BASE_SERIALIZE_METHODS(cls) \
257  FORMATTER_METHODS_PARAMS(cls, obj, paramcls, paramobj)
258 
259 // Templates for serializing to anything that looks like a stream,
260 // i.e. anything that supports .read(Span<std::byte>) and .write(Span<const std::byte>)
261 //
262 // clang-format off
263 
264 // Typically int8_t and char are distinct types, but some systems may define int8_t
265 // in terms of char. Forbid serialization of char in the typical case, but allow it if
266 // it's the only way to describe an int8_t.
267 template<class T>
268 concept CharNotInt8 = std::same_as<T, char> && !std::same_as<T, int8_t>;
269 
270 template <typename Stream, CharNotInt8 V> void Serialize(Stream&, V) = delete; // char serialization forbidden. Use uint8_t or int8_t
271 template <typename Stream> void Serialize(Stream& s, std::byte a) { ser_writedata8(s, uint8_t(a)); }
272 template<typename Stream> inline void Serialize(Stream& s, int8_t a ) { ser_writedata8(s, a); }
273 template<typename Stream> inline void Serialize(Stream& s, uint8_t a ) { ser_writedata8(s, a); }
274 template<typename Stream> inline void Serialize(Stream& s, int16_t a ) { ser_writedata16(s, a); }
275 template<typename Stream> inline void Serialize(Stream& s, uint16_t a) { ser_writedata16(s, a); }
276 template<typename Stream> inline void Serialize(Stream& s, int32_t a ) { ser_writedata32(s, a); }
277 template<typename Stream> inline void Serialize(Stream& s, uint32_t a) { ser_writedata32(s, a); }
278 template<typename Stream> inline void Serialize(Stream& s, int64_t a ) { ser_writedata64(s, a); }
279 template<typename Stream> inline void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); }
280 template <typename Stream, BasicByte B, int N> void Serialize(Stream& s, const B (&a)[N]) { s.write(MakeByteSpan(a)); }
281 template <typename Stream, BasicByte B, std::size_t N> void Serialize(Stream& s, const std::array<B, N>& a) { s.write(MakeByteSpan(a)); }
282 template <typename Stream, BasicByte B> void Serialize(Stream& s, Span<B> span) { s.write(AsBytes(span)); }
283 
284 template <typename Stream, CharNotInt8 V> void Unserialize(Stream&, V) = delete; // char serialization forbidden. Use uint8_t or int8_t
285 template <typename Stream> void Unserialize(Stream& s, std::byte& a) { a = std::byte{ser_readdata8(s)}; }
286 template<typename Stream> inline void Unserialize(Stream& s, int8_t& a ) { a = ser_readdata8(s); }
287 template<typename Stream> inline void Unserialize(Stream& s, uint8_t& a ) { a = ser_readdata8(s); }
288 template<typename Stream> inline void Unserialize(Stream& s, int16_t& a ) { a = ser_readdata16(s); }
289 template<typename Stream> inline void Unserialize(Stream& s, uint16_t& a) { a = ser_readdata16(s); }
290 template<typename Stream> inline void Unserialize(Stream& s, int32_t& a ) { a = ser_readdata32(s); }
291 template<typename Stream> inline void Unserialize(Stream& s, uint32_t& a) { a = ser_readdata32(s); }
292 template<typename Stream> inline void Unserialize(Stream& s, int64_t& a ) { a = ser_readdata64(s); }
293 template<typename Stream> inline void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); }
294 template <typename Stream, BasicByte B, int N> void Unserialize(Stream& s, B (&a)[N]) { s.read(MakeWritableByteSpan(a)); }
295 template <typename Stream, BasicByte B, std::size_t N> void Unserialize(Stream& s, std::array<B, N>& a) { s.read(MakeWritableByteSpan(a)); }
296 template <typename Stream, BasicByte B> void Unserialize(Stream& s, Span<B> span) { s.read(AsWritableBytes(span)); }
297 
298 template <typename Stream> inline void Serialize(Stream& s, bool a) { uint8_t f = a; ser_writedata8(s, f); }
299 template <typename Stream> inline void Unserialize(Stream& s, bool& a) { uint8_t f = ser_readdata8(s); a = f; }
300 // clang-format on
301 
302 
310 constexpr inline unsigned int GetSizeOfCompactSize(uint64_t nSize)
311 {
312  if (nSize < 253) return sizeof(unsigned char);
313  else if (nSize <= std::numeric_limits<uint16_t>::max()) return sizeof(unsigned char) + sizeof(uint16_t);
314  else if (nSize <= std::numeric_limits<unsigned int>::max()) return sizeof(unsigned char) + sizeof(unsigned int);
315  else return sizeof(unsigned char) + sizeof(uint64_t);
316 }
317 
318 inline void WriteCompactSize(SizeComputer& os, uint64_t nSize);
319 
320 template<typename Stream>
321 void WriteCompactSize(Stream& os, uint64_t nSize)
322 {
323  if (nSize < 253)
324  {
325  ser_writedata8(os, nSize);
326  }
327  else if (nSize <= std::numeric_limits<uint16_t>::max())
328  {
329  ser_writedata8(os, 253);
330  ser_writedata16(os, nSize);
331  }
332  else if (nSize <= std::numeric_limits<unsigned int>::max())
333  {
334  ser_writedata8(os, 254);
335  ser_writedata32(os, nSize);
336  }
337  else
338  {
339  ser_writedata8(os, 255);
340  ser_writedata64(os, nSize);
341  }
342  return;
343 }
344 
351 template<typename Stream>
352 uint64_t ReadCompactSize(Stream& is, bool range_check = true)
353 {
354  uint8_t chSize = ser_readdata8(is);
355  uint64_t nSizeRet = 0;
356  if (chSize < 253)
357  {
358  nSizeRet = chSize;
359  }
360  else if (chSize == 253)
361  {
362  nSizeRet = ser_readdata16(is);
363  if (nSizeRet < 253)
364  throw std::ios_base::failure("non-canonical ReadCompactSize()");
365  }
366  else if (chSize == 254)
367  {
368  nSizeRet = ser_readdata32(is);
369  if (nSizeRet < 0x10000u)
370  throw std::ios_base::failure("non-canonical ReadCompactSize()");
371  }
372  else
373  {
374  nSizeRet = ser_readdata64(is);
375  if (nSizeRet < 0x100000000ULL)
376  throw std::ios_base::failure("non-canonical ReadCompactSize()");
377  }
378  if (range_check && nSizeRet > MAX_SIZE) {
379  throw std::ios_base::failure("ReadCompactSize(): size too large");
380  }
381  return nSizeRet;
382 }
383 
419 
420 template <VarIntMode Mode, typename I>
422  constexpr CheckVarIntMode()
423  {
424  static_assert(Mode != VarIntMode::DEFAULT || std::is_unsigned<I>::value, "Unsigned type required with mode DEFAULT.");
425  static_assert(Mode != VarIntMode::NONNEGATIVE_SIGNED || std::is_signed<I>::value, "Signed type required with mode NONNEGATIVE_SIGNED.");
426  }
427 };
428 
429 template<VarIntMode Mode, typename I>
430 inline unsigned int GetSizeOfVarInt(I n)
431 {
433  int nRet = 0;
434  while(true) {
435  nRet++;
436  if (n <= 0x7F)
437  break;
438  n = (n >> 7) - 1;
439  }
440  return nRet;
441 }
442 
443 template<typename I>
444 inline void WriteVarInt(SizeComputer& os, I n);
445 
446 template<typename Stream, VarIntMode Mode, typename I>
447 void WriteVarInt(Stream& os, I n)
448 {
450  unsigned char tmp[(sizeof(n)*8+6)/7];
451  int len=0;
452  while(true) {
453  tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00);
454  if (n <= 0x7F)
455  break;
456  n = (n >> 7) - 1;
457  len++;
458  }
459  do {
460  ser_writedata8(os, tmp[len]);
461  } while(len--);
462 }
463 
464 template<typename Stream, VarIntMode Mode, typename I>
465 I ReadVarInt(Stream& is)
466 {
468  I n = 0;
469  while(true) {
470  unsigned char chData = ser_readdata8(is);
471  if (n > (std::numeric_limits<I>::max() >> 7)) {
472  throw std::ios_base::failure("ReadVarInt(): size too large");
473  }
474  n = (n << 7) | (chData & 0x7F);
475  if (chData & 0x80) {
476  if (n == std::numeric_limits<I>::max()) {
477  throw std::ios_base::failure("ReadVarInt(): size too large");
478  }
479  n++;
480  } else {
481  return n;
482  }
483  }
484 }
485 
487 template<typename Formatter, typename T>
488 class Wrapper
489 {
490  static_assert(std::is_lvalue_reference<T>::value, "Wrapper needs an lvalue reference type T");
491 protected:
493 public:
494  explicit Wrapper(T obj) : m_object(obj) {}
495  template<typename Stream> void Serialize(Stream &s) const { Formatter().Ser(s, m_object); }
496  template<typename Stream> void Unserialize(Stream &s) { Formatter().Unser(s, m_object); }
497 };
498 
509 template<typename Formatter, typename T>
510 static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T&>(t); }
511 
512 #define VARINT_MODE(obj, mode) Using<VarIntFormatter<mode>>(obj)
513 #define VARINT(obj) Using<VarIntFormatter<VarIntMode::DEFAULT>>(obj)
514 #define COMPACTSIZE(obj) Using<CompactSizeFormatter<true>>(obj)
515 #define LIMITED_STRING(obj,n) Using<LimitedStringFormatter<n>>(obj)
516 
518 template<VarIntMode Mode>
520 {
521  template<typename Stream, typename I> void Ser(Stream &s, I v)
522  {
523  WriteVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s, v);
524  }
525 
526  template<typename Stream, typename I> void Unser(Stream& s, I& v)
527  {
528  v = ReadVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s);
529  }
530 };
531 
541 template<int Bytes, bool BigEndian = false>
543 {
544  static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range");
545  static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes));
546 
547  template <typename Stream, typename I> void Ser(Stream& s, I v)
548  {
549  if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range");
550  if (BigEndian) {
551  uint64_t raw = htobe64_internal(v);
552  s.write(AsBytes(Span{&raw, 1}).last(Bytes));
553  } else {
554  uint64_t raw = htole64_internal(v);
555  s.write(AsBytes(Span{&raw, 1}).first(Bytes));
556  }
557  }
558 
559  template <typename Stream, typename I> void Unser(Stream& s, I& v)
560  {
561  using U = typename std::conditional<std::is_enum<I>::value, std::underlying_type<I>, std::common_type<I>>::type::type;
562  static_assert(std::numeric_limits<U>::max() >= MAX && std::numeric_limits<U>::min() <= 0, "Assigned type too small");
563  uint64_t raw = 0;
564  if (BigEndian) {
565  s.read(AsWritableBytes(Span{&raw, 1}).last(Bytes));
566  v = static_cast<I>(be64toh_internal(raw));
567  } else {
568  s.read(AsWritableBytes(Span{&raw, 1}).first(Bytes));
569  v = static_cast<I>(le64toh_internal(raw));
570  }
571  }
572 };
573 
575 
577 template<bool RangeCheck>
579 {
580  template<typename Stream, typename I>
581  void Unser(Stream& s, I& v)
582  {
583  uint64_t n = ReadCompactSize<Stream>(s, RangeCheck);
584  if (n < std::numeric_limits<I>::min() || n > std::numeric_limits<I>::max()) {
585  throw std::ios_base::failure("CompactSize exceeds limit of type");
586  }
587  v = n;
588  }
589 
590  template<typename Stream, typename I>
591  void Ser(Stream& s, I v)
592  {
593  static_assert(std::is_unsigned<I>::value, "CompactSize only supported for unsigned integers");
594  static_assert(std::numeric_limits<I>::max() <= std::numeric_limits<uint64_t>::max(), "CompactSize only supports 64-bit integers and below");
595 
596  WriteCompactSize<Stream>(s, v);
597  }
598 };
599 
600 template <typename U, bool LOSSY = false>
602  template <typename Stream, typename Tp>
603  void Unser(Stream& s, Tp& tp)
604  {
605  U u;
606  s >> u;
607  // Lossy deserialization does not make sense, so force Wnarrowing
608  tp = Tp{typename Tp::duration{typename Tp::duration::rep{u}}};
609  }
610  template <typename Stream, typename Tp>
611  void Ser(Stream& s, Tp tp)
612  {
613  if constexpr (LOSSY) {
614  s << U(tp.time_since_epoch().count());
615  } else {
616  s << U{tp.time_since_epoch().count()};
617  }
618  }
619 };
620 template <typename U>
622 
624 {
625 protected:
626  uint64_t n;
627 public:
628  explicit CompactSizeWriter(uint64_t n_in) : n(n_in) { }
629 
630  template<typename Stream>
631  void Serialize(Stream &s) const {
632  WriteCompactSize<Stream>(s, n);
633  }
634 };
635 
636 template<size_t Limit>
638 {
639  template<typename Stream>
640  void Unser(Stream& s, std::string& v)
641  {
642  size_t size = ReadCompactSize(s);
643  if (size > Limit) {
644  throw std::ios_base::failure("String length limit exceeded");
645  }
646  v.resize(size);
647  if (size != 0) s.read(MakeWritableByteSpan(v));
648  }
649 
650  template<typename Stream>
651  void Ser(Stream& s, const std::string& v)
652  {
653  s << v;
654  }
655 };
656 
670 template<class Formatter>
672 {
673  template<typename Stream, typename V>
674  void Ser(Stream& s, const V& v)
675  {
676  Formatter formatter;
677  WriteCompactSize(s, v.size());
678  for (const typename V::value_type& elem : v) {
679  formatter.Ser(s, elem);
680  }
681  }
682 
683  template<typename Stream, typename V>
684  void Unser(Stream& s, V& v)
685  {
686  Formatter formatter;
687  v.clear();
688  size_t size = ReadCompactSize(s);
689  size_t allocated = 0;
690  while (allocated < size) {
691  // For DoS prevention, do not blindly allocate as much as the stream claims to contain.
692  // Instead, allocate in 5MiB batches, so that an attacker actually needs to provide
693  // X MiB of data to make us allocate X+5 Mib.
694  static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE, "Vector element size too large");
695  allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type));
696  v.reserve(allocated);
697  while (v.size() < allocated) {
698  v.emplace_back();
699  formatter.Unser(s, v.back());
700  }
701  }
702  };
703 };
704 
712 template<typename Stream, typename C> void Serialize(Stream& os, const std::basic_string<C>& str);
713 template<typename Stream, typename C> void Unserialize(Stream& is, std::basic_string<C>& str);
714 
718 template<typename Stream, unsigned int N, typename T> inline void Serialize(Stream& os, const prevector<N, T>& v);
719 template<typename Stream, unsigned int N, typename T> inline void Unserialize(Stream& is, prevector<N, T>& v);
720 
724 template<typename Stream, typename T, typename A> inline void Serialize(Stream& os, const std::vector<T, A>& v);
725 template<typename Stream, typename T, typename A> inline void Unserialize(Stream& is, std::vector<T, A>& v);
726 
730 template<typename Stream, typename K, typename T> void Serialize(Stream& os, const std::pair<K, T>& item);
731 template<typename Stream, typename K, typename T> void Unserialize(Stream& is, std::pair<K, T>& item);
732 
736 template<typename Stream, typename K, typename T, typename Pred, typename A> void Serialize(Stream& os, const std::map<K, T, Pred, A>& m);
737 template<typename Stream, typename K, typename T, typename Pred, typename A> void Unserialize(Stream& is, std::map<K, T, Pred, A>& m);
738 
742 template<typename Stream, typename K, typename Pred, typename A> void Serialize(Stream& os, const std::set<K, Pred, A>& m);
743 template<typename Stream, typename K, typename Pred, typename A> void Unserialize(Stream& is, std::set<K, Pred, A>& m);
744 
748 template<typename Stream, typename T> void Serialize(Stream& os, const std::shared_ptr<const T>& p);
749 template<typename Stream, typename T> void Unserialize(Stream& os, std::shared_ptr<const T>& p);
750 
754 template<typename Stream, typename T> void Serialize(Stream& os, const std::unique_ptr<const T>& p);
755 template<typename Stream, typename T> void Unserialize(Stream& os, std::unique_ptr<const T>& p);
756 
757 
761 template <class T, class Stream>
762 concept Serializable = requires(T a, Stream s) { a.Serialize(s); };
763 template <typename Stream, typename T>
764  requires Serializable<T, Stream>
765 void Serialize(Stream& os, const T& a)
766 {
767  a.Serialize(os);
768 }
769 
770 template <class T, class Stream>
771 concept Unserializable = requires(T a, Stream s) { a.Unserialize(s); };
772 template <typename Stream, typename T>
773  requires Unserializable<T, Stream>
774 void Unserialize(Stream& is, T&& a)
775 {
776  a.Unserialize(is);
777 }
778 
785 {
786  template<typename Stream, typename T>
787  static void Ser(Stream& s, const T& t) { Serialize(s, t); }
788 
789  template<typename Stream, typename T>
790  static void Unser(Stream& s, T& t) { Unserialize(s, t); }
791 };
792 
793 
794 
795 
796 
800 template<typename Stream, typename C>
801 void Serialize(Stream& os, const std::basic_string<C>& str)
802 {
803  WriteCompactSize(os, str.size());
804  if (!str.empty())
805  os.write(MakeByteSpan(str));
806 }
807 
808 template<typename Stream, typename C>
809 void Unserialize(Stream& is, std::basic_string<C>& str)
810 {
811  unsigned int nSize = ReadCompactSize(is);
812  str.resize(nSize);
813  if (nSize != 0)
814  is.read(MakeWritableByteSpan(str));
815 }
816 
817 
818 
822 template <typename Stream, unsigned int N, typename T>
823 void Serialize(Stream& os, const prevector<N, T>& v)
824 {
825  if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes
826  WriteCompactSize(os, v.size());
827  if (!v.empty()) os.write(MakeByteSpan(v));
828  } else {
830  }
831 }
832 
833 
834 template <typename Stream, unsigned int N, typename T>
835 void Unserialize(Stream& is, prevector<N, T>& v)
836 {
837  if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes
838  // Limit size per read so bogus size value won't cause out of memory
839  v.clear();
840  unsigned int nSize = ReadCompactSize(is);
841  unsigned int i = 0;
842  while (i < nSize) {
843  unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T)));
844  v.resize_uninitialized(i + blk);
845  is.read(AsWritableBytes(Span{&v[i], blk}));
846  i += blk;
847  }
848  } else {
850  }
851 }
852 
853 
857 template <typename Stream, typename T, typename A>
858 void Serialize(Stream& os, const std::vector<T, A>& v)
859 {
860  if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes
861  WriteCompactSize(os, v.size());
862  if (!v.empty()) os.write(MakeByteSpan(v));
863  } else if constexpr (std::is_same_v<T, bool>) {
864  // A special case for std::vector<bool>, as dereferencing
865  // std::vector<bool>::const_iterator does not result in a const bool&
866  // due to std::vector's special casing for bool arguments.
867  WriteCompactSize(os, v.size());
868  for (bool elem : v) {
869  ::Serialize(os, elem);
870  }
871  } else {
873  }
874 }
875 
876 
877 template <typename Stream, typename T, typename A>
878 void Unserialize(Stream& is, std::vector<T, A>& v)
879 {
880  if constexpr (BasicByte<T>) { // Use optimized version for unformatted basic bytes
881  // Limit size per read so bogus size value won't cause out of memory
882  v.clear();
883  unsigned int nSize = ReadCompactSize(is);
884  unsigned int i = 0;
885  while (i < nSize) {
886  unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T)));
887  v.resize(i + blk);
888  is.read(AsWritableBytes(Span{&v[i], blk}));
889  i += blk;
890  }
891  } else {
893  }
894 }
895 
896 
900 template<typename Stream, typename K, typename T>
901 void Serialize(Stream& os, const std::pair<K, T>& item)
902 {
903  Serialize(os, item.first);
904  Serialize(os, item.second);
905 }
906 
907 template<typename Stream, typename K, typename T>
908 void Unserialize(Stream& is, std::pair<K, T>& item)
909 {
910  Unserialize(is, item.first);
911  Unserialize(is, item.second);
912 }
913 
914 
915 
919 template<typename Stream, typename K, typename T, typename Pred, typename A>
920 void Serialize(Stream& os, const std::map<K, T, Pred, A>& m)
921 {
922  WriteCompactSize(os, m.size());
923  for (const auto& entry : m)
924  Serialize(os, entry);
925 }
926 
927 template<typename Stream, typename K, typename T, typename Pred, typename A>
928 void Unserialize(Stream& is, std::map<K, T, Pred, A>& m)
929 {
930  m.clear();
931  unsigned int nSize = ReadCompactSize(is);
932  typename std::map<K, T, Pred, A>::iterator mi = m.begin();
933  for (unsigned int i = 0; i < nSize; i++)
934  {
935  std::pair<K, T> item;
936  Unserialize(is, item);
937  mi = m.insert(mi, item);
938  }
939 }
940 
941 
942 
946 template<typename Stream, typename K, typename Pred, typename A>
947 void Serialize(Stream& os, const std::set<K, Pred, A>& m)
948 {
949  WriteCompactSize(os, m.size());
950  for (typename std::set<K, Pred, A>::const_iterator it = m.begin(); it != m.end(); ++it)
951  Serialize(os, (*it));
952 }
953 
954 template<typename Stream, typename K, typename Pred, typename A>
955 void Unserialize(Stream& is, std::set<K, Pred, A>& m)
956 {
957  m.clear();
958  unsigned int nSize = ReadCompactSize(is);
959  typename std::set<K, Pred, A>::iterator it = m.begin();
960  for (unsigned int i = 0; i < nSize; i++)
961  {
962  K key;
963  Unserialize(is, key);
964  it = m.insert(it, key);
965  }
966 }
967 
968 
969 
973 template<typename Stream, typename T> void
974 Serialize(Stream& os, const std::unique_ptr<const T>& p)
975 {
976  Serialize(os, *p);
977 }
978 
979 template<typename Stream, typename T>
980 void Unserialize(Stream& is, std::unique_ptr<const T>& p)
981 {
982  p.reset(new T(deserialize, is));
983 }
984 
985 
986 
990 template<typename Stream, typename T> void
991 Serialize(Stream& os, const std::shared_ptr<const T>& p)
992 {
993  Serialize(os, *p);
994 }
995 
996 template<typename Stream, typename T>
997 void Unserialize(Stream& is, std::shared_ptr<const T>& p)
998 {
999  p = std::make_shared<const T>(deserialize, is);
1000 }
1001 
1006 template <typename Stream, typename... Args>
1007 void SerializeMany(Stream& s, const Args&... args)
1008 {
1009  (::Serialize(s, args), ...);
1010 }
1011 
1012 template <typename Stream, typename... Args>
1013 inline void UnserializeMany(Stream& s, Args&&... args)
1014 {
1015  (::Unserialize(s, args), ...);
1016 }
1017 
1022  static constexpr bool ForRead() { return false; }
1023 
1024  template<typename Stream, typename... Args>
1025  static void SerReadWriteMany(Stream& s, const Args&... args)
1026  {
1027  ::SerializeMany(s, args...);
1028  }
1029 
1030  template<typename Stream, typename Type, typename Fn>
1031  static void SerRead(Stream& s, Type&&, Fn&&)
1032  {
1033  }
1034 
1035  template<typename Stream, typename Type, typename Fn>
1036  static void SerWrite(Stream& s, Type&& obj, Fn&& fn)
1037  {
1038  fn(s, std::forward<Type>(obj));
1039  }
1040 };
1042  static constexpr bool ForRead() { return true; }
1043 
1044  template<typename Stream, typename... Args>
1045  static void SerReadWriteMany(Stream& s, Args&&... args)
1046  {
1047  ::UnserializeMany(s, args...);
1048  }
1049 
1050  template<typename Stream, typename Type, typename Fn>
1051  static void SerRead(Stream& s, Type&& obj, Fn&& fn)
1052  {
1053  fn(s, std::forward<Type>(obj));
1054  }
1055 
1056  template<typename Stream, typename Type, typename Fn>
1057  static void SerWrite(Stream& s, Type&&, Fn&&)
1058  {
1059  }
1060 };
1061 
1062 /* ::GetSerializeSize implementations
1063  *
1064  * Computing the serialized size of objects is done through a special stream
1065  * object of type SizeComputer, which only records the number of bytes written
1066  * to it.
1067  *
1068  * If your Serialize or SerializationOp method has non-trivial overhead for
1069  * serialization, it may be worthwhile to implement a specialized version for
1070  * SizeComputer, which uses the s.seek() method to record bytes that would
1071  * be written instead.
1072  */
1074 {
1075 protected:
1076  size_t nSize{0};
1077 
1078 public:
1080 
1082  {
1083  this->nSize += src.size();
1084  }
1085 
1087  void seek(size_t _nSize)
1088  {
1089  this->nSize += _nSize;
1090  }
1091 
1092  template<typename T>
1094  {
1095  ::Serialize(*this, obj);
1096  return (*this);
1097  }
1098 
1099  size_t size() const {
1100  return nSize;
1101  }
1102 };
1103 
1104 template<typename I>
1105 inline void WriteVarInt(SizeComputer &s, I n)
1106 {
1107  s.seek(GetSizeOfVarInt<I>(n));
1108 }
1109 
1110 inline void WriteCompactSize(SizeComputer &s, uint64_t nSize)
1111 {
1112  s.seek(GetSizeOfCompactSize(nSize));
1113 }
1114 
1115 template <typename T>
1116 size_t GetSerializeSize(const T& t)
1117 {
1118  return (SizeComputer() << t).size();
1119 }
1120 
1122 template <typename Params, typename SubStream>
1124 {
1126  SubStream& m_substream; // private to avoid leaking version/type into serialization code that shouldn't see it
1127 
1128 public:
1129  ParamsStream(const Params& params LIFETIMEBOUND, SubStream& substream LIFETIMEBOUND) : m_params{params}, m_substream{substream} {}
1130  template <typename U> ParamsStream& operator<<(const U& obj) { ::Serialize(*this, obj); return *this; }
1131  template <typename U> ParamsStream& operator>>(U&& obj) { ::Unserialize(*this, obj); return *this; }
1132  void write(Span<const std::byte> src) { m_substream.write(src); }
1133  void read(Span<std::byte> dst) { m_substream.read(dst); }
1134  void ignore(size_t num) { m_substream.ignore(num); }
1135  bool eof() const { return m_substream.eof(); }
1136  size_t size() const { return m_substream.size(); }
1137  const Params& GetParams() const { return m_params; }
1138  int GetVersion() = delete; // Deprecated with Params usage
1139  int GetType() = delete; // Deprecated with Params usage
1140 };
1141 
1143 template <typename Params, typename T>
1145 {
1148 
1149 public:
1150  explicit ParamsWrapper(const Params& params, T& obj) : m_params{params}, m_object{obj} {}
1151 
1152  template <typename Stream>
1153  void Serialize(Stream& s) const
1154  {
1155  ParamsStream ss{m_params, s};
1156  ::Serialize(ss, m_object);
1157  }
1158  template <typename Stream>
1159  void Unserialize(Stream& s)
1160  {
1161  ParamsStream ss{m_params, s};
1162  ::Unserialize(ss, m_object);
1163  }
1164 };
1165 
1175 #define SER_PARAMS_OPFUNC \
1176  \
1181  template <typename T> \
1182  auto operator()(T&& t) const \
1183  { \
1184  return ParamsWrapper{*this, t}; \
1185  }
1186 
1187 #endif // BITCOIN_SERIALIZE_H
#define LIFETIMEBOUND
Definition: attributes.h:16
ArgsManager & args
Definition: bitcoind.cpp:268
const CChainParams & Params()
Return the currently selected parameters.
void Serialize(Stream &s) const
Definition: serialize.h:631
CompactSizeWriter(uint64_t n_in)
Definition: serialize.h:628
Wrapper that overrides the GetParams() function of a stream (and hides GetVersion/GetType).
Definition: serialize.h:1124
const Params & m_params
Definition: serialize.h:1125
void ignore(size_t num)
Definition: serialize.h:1134
ParamsStream(const Params &params LIFETIMEBOUND, SubStream &substream LIFETIMEBOUND)
Definition: serialize.h:1129
void write(Span< const std::byte > src)
Definition: serialize.h:1132
int GetVersion()=delete
void read(Span< std::byte > dst)
Definition: serialize.h:1133
bool eof() const
Definition: serialize.h:1135
ParamsStream & operator>>(U &&obj)
Definition: serialize.h:1131
const Params & GetParams() const
Definition: serialize.h:1137
ParamsStream & operator<<(const U &obj)
Definition: serialize.h:1130
size_t size() const
Definition: serialize.h:1136
SubStream & m_substream
Definition: serialize.h:1126
int GetType()=delete
Wrapper that serializes objects with the specified parameters.
Definition: serialize.h:1145
const Params & m_params
Definition: serialize.h:1146
void Unserialize(Stream &s)
Definition: serialize.h:1159
void Serialize(Stream &s) const
Definition: serialize.h:1153
ParamsWrapper(const Params &params, T &obj)
Definition: serialize.h:1150
void write(Span< const std::byte > src)
Definition: serialize.h:1081
void seek(size_t _nSize)
Pretend _nSize bytes are written, without specifying them.
Definition: serialize.h:1087
SizeComputer & operator<<(const T &obj)
Definition: serialize.h:1093
size_t nSize
Definition: serialize.h:1076
size_t size() const
Definition: serialize.h:1099
A Span is an object that can refer to a contiguous sequence of objects.
Definition: span.h:98
constexpr std::size_t size() const noexcept
Definition: span.h:187
Simple wrapper class to serialize objects using a formatter; used by Using().
Definition: serialize.h:489
Wrapper(T obj)
Definition: serialize.h:494
T m_object
Definition: serialize.h:490
void Serialize(Stream &s) const
Definition: serialize.h:495
void Unserialize(Stream &s)
Definition: serialize.h:496
Implements a drop-in replacement for std::vector<T> which stores up to N elements directly (without h...
Definition: prevector.h:37
bool empty() const
Definition: prevector.h:300
void clear()
Definition: prevector.h:357
size_type size() const
Definition: prevector.h:296
void resize_uninitialized(size_type new_size)
Definition: prevector.h:400
BSWAP_CONSTEXPR uint32_t be32toh_internal(uint32_t big_endian_32bits)
Definition: endian.h:43
BSWAP_CONSTEXPR uint16_t be16toh_internal(uint16_t big_endian_16bits)
Definition: endian.h:23
BSWAP_CONSTEXPR uint64_t htobe64_internal(uint64_t host_64bits)
Definition: endian.h:53
BSWAP_CONSTEXPR uint16_t htobe16_internal(uint16_t host_16bits)
Definition: endian.h:13
BSWAP_CONSTEXPR uint32_t htole32_internal(uint32_t host_32bits)
Definition: endian.h:38
BSWAP_CONSTEXPR uint16_t htole16_internal(uint16_t host_16bits)
Definition: endian.h:18
BSWAP_CONSTEXPR uint64_t be64toh_internal(uint64_t big_endian_64bits)
Definition: endian.h:63
BSWAP_CONSTEXPR uint16_t le16toh_internal(uint16_t little_endian_16bits)
Definition: endian.h:28
BSWAP_CONSTEXPR uint64_t htole64_internal(uint64_t host_64bits)
Definition: endian.h:58
BSWAP_CONSTEXPR uint64_t le64toh_internal(uint64_t little_endian_64bits)
Definition: endian.h:68
BSWAP_CONSTEXPR uint32_t le32toh_internal(uint32_t little_endian_32bits)
Definition: endian.h:48
BSWAP_CONSTEXPR uint32_t htobe32_internal(uint32_t host_32bits)
Definition: endian.h:33
#define T(expected, seed, data)
size_t GetSerializeSize(const T &t)
Definition: serialize.h:1116
void Serialize(Stream &, V)=delete
constexpr unsigned int GetSizeOfCompactSize(uint64_t nSize)
Compact Size size < 253 – 1 byte size <= USHRT_MAX – 3 bytes (253 + 2 bytes) size <= UINT_MAX – 5 byt...
Definition: serialize.h:310
void SerializeMany(Stream &s, const Args &... args)
Support for (un)serializing many things at once.
Definition: serialize.h:1007
static const unsigned int MAX_VECTOR_ALLOCATE
Maximum amount of memory (in bytes) to allocate at once when deserializing vectors.
Definition: serialize.h:35
uint8_t ser_readdata8(Stream &s)
Definition: serialize.h:83
I ReadVarInt(Stream &is)
Definition: serialize.h:465
void ser_writedata32be(Stream &s, uint32_t obj)
Definition: serialize.h:73
VarIntMode
Variable-length integers: bytes are a MSB base-128 encoding of the number.
Definition: serialize.h:418
@ NONNEGATIVE_SIGNED
void ser_writedata32(Stream &s, uint32_t obj)
Definition: serialize.h:68
static constexpr uint64_t MAX_SIZE
The maximum size of a serialized object in bytes or number of elements (for eg vectors) when the size...
Definition: serialize.h:32
constexpr deserialize_type deserialize
Definition: serialize.h:49
void ser_writedata16(Stream &s, uint16_t obj)
Definition: serialize.h:58
Out & AsBase(In &x)
Convert any argument to a reference to X, maintaining constness.
Definition: serialize.h:144
void Unserialize(Stream &, V)=delete
concept CharNotInt8
Definition: serialize.h:268
void UnserializeMany(Stream &s, Args &&... args)
Definition: serialize.h:1013
static Wrapper< Formatter, T & > Using(T &&t)
Cause serialization/deserialization of an object to be done using a specified formatter class.
Definition: serialize.h:510
void WriteVarInt(SizeComputer &os, I n)
Definition: serialize.h:1105
void WriteCompactSize(SizeComputer &os, uint64_t nSize)
Definition: serialize.h:1110
concept Unserializable
Definition: serialize.h:771
void ser_writedata16be(Stream &s, uint16_t obj)
Definition: serialize.h:63
uint16_t ser_readdata16(Stream &s)
Definition: serialize.h:89
uint64_t ser_readdata64(Stream &s)
Definition: serialize.h:113
void ser_writedata8(Stream &s, uint8_t obj)
Definition: serialize.h:54
uint64_t ReadCompactSize(Stream &is, bool range_check=true)
Decode a CompactSize-encoded variable-length integer.
Definition: serialize.h:352
unsigned int GetSizeOfVarInt(I n)
Definition: serialize.h:430
concept Serializable
If none of the specialized versions above matched, default to calling member function.
Definition: serialize.h:762
uint32_t ser_readdata32(Stream &s)
Definition: serialize.h:101
uint16_t ser_readdata16be(Stream &s)
Definition: serialize.h:95
void ser_writedata64(Stream &s, uint64_t obj)
Definition: serialize.h:78
uint32_t ser_readdata32be(Stream &s)
Definition: serialize.h:107
Span< std::byte > AsWritableBytes(Span< T > s) noexcept
Definition: span.h:271
Span< const std::byte > MakeByteSpan(V &&v) noexcept
Definition: span.h:277
Span< const std::byte > AsBytes(Span< T > s) noexcept
Definition: span.h:266
Span< std::byte > MakeWritableByteSpan(V &&v) noexcept
Definition: span.h:282
Support for all macros providing or using the ser_action parameter of the SerializationOps method.
Definition: serialize.h:1021
static void SerReadWriteMany(Stream &s, const Args &... args)
Definition: serialize.h:1025
static constexpr bool ForRead()
Definition: serialize.h:1022
static void SerWrite(Stream &s, Type &&obj, Fn &&fn)
Definition: serialize.h:1036
static void SerRead(Stream &s, Type &&, Fn &&)
Definition: serialize.h:1031
static constexpr bool ForRead()
Definition: serialize.h:1042
static void SerReadWriteMany(Stream &s, Args &&... args)
Definition: serialize.h:1045
static void SerRead(Stream &s, Type &&obj, Fn &&fn)
Definition: serialize.h:1051
static void SerWrite(Stream &s, Type &&, Fn &&)
Definition: serialize.h:1057
constexpr CheckVarIntMode()
Definition: serialize.h:422
void Unser(Stream &s, Tp &tp)
Definition: serialize.h:603
void Ser(Stream &s, Tp tp)
Definition: serialize.h:611
Formatter for integers in CompactSize format.
Definition: serialize.h:579
void Unser(Stream &s, I &v)
Definition: serialize.h:581
void Ser(Stream &s, I v)
Definition: serialize.h:591
Serialization wrapper class for custom integers and enums.
Definition: serialize.h:543
static constexpr uint64_t MAX
Definition: serialize.h:545
void Ser(Stream &s, I v)
Definition: serialize.h:547
void Unser(Stream &s, I &v)
Definition: serialize.h:559
Default formatter.
Definition: serialize.h:785
static void Ser(Stream &s, const T &t)
Definition: serialize.h:787
static void Unser(Stream &s, T &t)
Definition: serialize.h:790
void Unser(Stream &s, std::string &v)
Definition: serialize.h:640
void Ser(Stream &s, const std::string &v)
Definition: serialize.h:651
Serialization wrapper class for integers in VarInt format.
Definition: serialize.h:520
void Ser(Stream &s, I v)
Definition: serialize.h:521
void Unser(Stream &s, I &v)
Definition: serialize.h:526
Formatter to serialize/deserialize vector elements using another formatter.
Definition: serialize.h:672
void Unser(Stream &s, V &v)
Definition: serialize.h:684
void Ser(Stream &s, const V &v)
Definition: serialize.h:674
Dummy data type to identify deserializing constructors.
Definition: serialize.h:48
#define B
Definition: util_tests.cpp:489