Bitcoin Core  26.99.0
P2P Digital Currency
blockstorage.h
Go to the documentation of this file.
1 // Copyright (c) 2011-2022 The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 
5 #ifndef BITCOIN_NODE_BLOCKSTORAGE_H
6 #define BITCOIN_NODE_BLOCKSTORAGE_H
7 
8 #include <attributes.h>
9 #include <chain.h>
10 #include <dbwrapper.h>
11 #include <flatfile.h>
13 #include <kernel/chainparams.h>
14 #include <kernel/cs_main.h>
16 #include <primitives/block.h>
17 #include <streams.h>
18 #include <sync.h>
19 #include <uint256.h>
20 #include <util/fs.h>
21 #include <util/hasher.h>
22 
23 #include <array>
24 #include <atomic>
25 #include <cstdint>
26 #include <functional>
27 #include <limits>
28 #include <map>
29 #include <memory>
30 #include <optional>
31 #include <set>
32 #include <string>
33 #include <unordered_map>
34 #include <utility>
35 #include <vector>
36 
38 class CBlockUndo;
39 class Chainstate;
40 class ChainstateManager;
41 namespace Consensus {
42 struct Params;
43 }
44 namespace util {
45 class SignalInterrupt;
46 } // namespace util
47 
48 namespace kernel {
50 class BlockTreeDB : public CDBWrapper
51 {
52 public:
54  bool WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*>>& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo);
55  bool ReadBlockFileInfo(int nFile, CBlockFileInfo& info);
56  bool ReadLastBlockFile(int& nFile);
57  bool WriteReindexing(bool fReindexing);
58  void ReadReindexing(bool& fReindexing);
59  bool WriteFlag(const std::string& name, bool fValue);
60  bool ReadFlag(const std::string& name, bool& fValue);
61  bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex, const util::SignalInterrupt& interrupt)
63 };
64 } // namespace kernel
65 
66 namespace node {
68 
70 static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB
72 static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB
74 static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB
75 
77 static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = std::tuple_size_v<MessageStartChars> + sizeof(unsigned int);
78 
79 extern std::atomic_bool fReindex;
80 
81 // Because validation code takes pointers to the map's CBlockIndex objects, if
82 // we ever switch to another associative container, we need to either use a
83 // container that has stable addressing (true of all std associative
84 // containers), or make the key a `std::unique_ptr<CBlockIndex>`
85 using BlockMap = std::unordered_map<uint256, CBlockIndex, BlockHasher>;
86 
88  bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const;
89 };
90 
92  /* Only compares the height of two block indices, doesn't try to tie-break */
93  bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const;
94 };
95 
96 struct PruneLockInfo {
97  int height_first{std::numeric_limits<int>::max()};
98 };
99 
101  // Values used as array indexes - do not change carelessly.
102  NORMAL = 0,
103  ASSUMED = 1,
105 };
106 
107 std::ostream& operator<<(std::ostream& os, const BlockfileType& type);
108 
110  // The latest blockfile number.
111  int file_num{0};
112 
113  // Track the height of the highest block in file_num whose undo
114  // data has been written. Block data is written to block files in download
115  // order, but is written to undo files in validation order, which is
116  // usually in order by height. To avoid wasting disk space, undo files will
117  // be trimmed whenever the corresponding block file is finalized and
118  // the height of the highest block written to the block file equals the
119  // height of the highest block written to the undo file. This is a
120  // heuristic and can sometimes preemptively trim undo files that will write
121  // more data later, and sometimes fail to trim undo files that can't have
122  // more data written later.
123  int undo_height{0};
124 };
125 
126 std::ostream& operator<<(std::ostream& os, const BlockfileCursor& cursor);
127 
128 
137 {
138  friend Chainstate;
140 
141 private:
142  const CChainParams& GetParams() const { return m_opts.chainparams; }
149  bool LoadBlockIndex(const std::optional<uint256>& snapshot_blockhash)
151 
153  [[nodiscard]] bool FlushBlockFile(int blockfile_num, bool fFinalize, bool finalize_undo);
154 
156  [[nodiscard]] bool FlushUndoFile(int block_file, bool finalize = false);
157 
158  [[nodiscard]] bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown);
159  [[nodiscard]] bool FlushChainstateBlockFile(int tip_height);
160  bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize);
161 
162  FlatFileSeq BlockFileSeq() const;
163  FlatFileSeq UndoFileSeq() const;
164 
165  AutoFile OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const;
166 
167  bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const;
168  bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const;
169 
170  /* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */
172  std::set<int>& setFilesToPrune,
173  int nManualPruneHeight,
174  const Chainstate& chain,
175  ChainstateManager& chainman);
176 
193  void FindFilesToPrune(
194  std::set<int>& setFilesToPrune,
195  int last_prune,
196  const Chainstate& chain,
197  ChainstateManager& chainman);
198 
200  std::vector<CBlockFileInfo> m_blockfile_info;
201 
212  std::array<std::optional<BlockfileCursor>, BlockfileType::NUM_TYPES>
213  m_blockfile_cursors GUARDED_BY(cs_LastBlockFile) = {
214  BlockfileCursor{},
215  std::nullopt,
216  };
218  {
219  static const BlockfileCursor empty_cursor;
220  const auto& normal = m_blockfile_cursors[BlockfileType::NORMAL].value_or(empty_cursor);
221  const auto& assumed = m_blockfile_cursors[BlockfileType::ASSUMED].value_or(empty_cursor);
222  return std::max(normal.file_num, assumed.file_num);
223  }
224 
229  bool m_check_for_pruning = false;
230 
231  const bool m_prune_mode;
232 
234  std::set<CBlockIndex*> m_dirty_blockindex;
235 
237  std::set<int> m_dirty_fileinfo;
238 
245  std::unordered_map<std::string, PruneLockInfo> m_prune_locks GUARDED_BY(::cs_main);
246 
248 
250 
251 public:
253 
254  explicit BlockManager(const util::SignalInterrupt& interrupt, Options opts)
255  : m_prune_mode{opts.prune_target > 0},
256  m_opts{std::move(opts)},
257  m_interrupt{interrupt} {};
258 
260  std::atomic<bool> m_importing{false};
261 
262  BlockMap m_block_index GUARDED_BY(cs_main);
263 
276  std::optional<int> m_snapshot_height;
277 
278  std::vector<CBlockIndex*> GetAllBlockIndices() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
279 
285 
286  std::unique_ptr<BlockTreeDB> m_block_tree_db GUARDED_BY(::cs_main);
287 
288  bool WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
289  bool LoadBlockIndexDB(const std::optional<uint256>& snapshot_blockhash)
291 
297  void ScanAndUnlinkAlreadyPrunedFiles() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
298 
302 
304  void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
305 
308 
310  CBlockFileInfo* GetBlockFileInfo(size_t n);
311 
312  bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
314 
316  FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, const FlatFilePos* dbp);
317 
319  [[nodiscard]] bool IsPruneMode() const { return m_prune_mode; }
320 
322  [[nodiscard]] uint64_t GetPruneTarget() const { return m_opts.prune_target; }
323  static constexpr auto PRUNE_TARGET_MANUAL{std::numeric_limits<uint64_t>::max()};
324 
325  [[nodiscard]] bool LoadingBlocks() const { return m_importing || fReindex; }
326 
328  uint64_t CalculateCurrentUsage();
329 
332 
336  bool CheckBlockDataAvailability(const CBlockIndex& upper_block LIFETIMEBOUND, const CBlockIndex& lower_block LIFETIMEBOUND) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
337 
341  const CBlockIndex* GetFirstStoredBlock(const CBlockIndex& start_block LIFETIMEBOUND, const CBlockIndex* lower_block=nullptr) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
342 
344  bool m_have_pruned = false;
345 
347  bool IsBlockPruned(const CBlockIndex& block) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
348 
350  void UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
351 
353  AutoFile OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false) const;
354 
356  fs::path GetBlockPosFilename(const FlatFilePos& pos) const;
357 
361  void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const;
362 
364  bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) const;
365  bool ReadBlockFromDisk(CBlock& block, const CBlockIndex& index) const;
366  bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos) const;
367 
368  bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& index) const;
369 
370  void CleanupBlockRevFiles() const;
371 };
372 
373 void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFiles);
374 } // namespace node
375 
376 #endif // BITCOIN_NODE_BLOCKSTORAGE_H
#define LIFETIMEBOUND
Definition: attributes.h:16
const CChainParams & Params()
Return the currently selected parameters.
Non-refcounted RAII wrapper for FILE*.
Definition: streams.h:389
Nodes collect new transactions into a block, hash them into a hash tree, and scan through nonce value...
Definition: block.h:22
Definition: block.h:69
The block chain is a tree shaped structure starting with the genesis block at the root,...
Definition: chain.h:150
Undo information for a CBlock.
Definition: undo.h:63
CChainParams defines various tweakable parameters of a given instance of the Bitcoin system.
Definition: chainparams.h:81
const Consensus::Params & GetConsensus() const
Definition: chainparams.h:93
CDBWrapper(const DBParams &params)
Definition: dbwrapper.cpp:221
Chainstate stores and provides an API to update our local knowledge of the current best chain.
Definition: validation.h:486
Provides an interface for creating and interacting with one or two chainstates: an IBD chainstate gen...
Definition: validation.h:844
FlatFileSeq represents a sequence of numbered files storing raw data.
Definition: flatfile.h:46
Access to the block database (blocks/index/)
Definition: blockstorage.h:51
bool ReadLastBlockFile(int &nFile)
bool ReadFlag(const std::string &name, bool &fValue)
bool ReadBlockFileInfo(int nFile, CBlockFileInfo &info)
bool WriteBatchSync(const std::vector< std::pair< int, const CBlockFileInfo * >> &fileInfo, int nLastFile, const std::vector< const CBlockIndex * > &blockinfo)
void ReadReindexing(bool &fReindexing)
bool WriteFlag(const std::string &name, bool fValue)
bool WriteReindexing(bool fReindexing)
Maintains a tree of blocks (stored in m_block_index) which is consulted to determine where the most-w...
Definition: blockstorage.h:137
const kernel::BlockManagerOpts m_opts
Definition: blockstorage.h:249
std::set< int > m_dirty_fileinfo
Dirty block file entries.
Definition: blockstorage.h:237
bool ReadBlockFromDisk(CBlock &block, const FlatFilePos &pos) const
Functions for disk access for blocks.
FlatFileSeq UndoFileSeq() const
RecursiveMutex cs_LastBlockFile
Definition: blockstorage.h:199
const Consensus::Params & GetConsensus() const
Definition: blockstorage.h:143
bool WriteUndoDataForBlock(const CBlockUndo &blockundo, BlockValidationState &state, CBlockIndex &block) EXCLUSIVE_LOCKS_REQUIRED(FlatFilePos SaveBlockToDisk(const CBlock &block, int nHeight, const FlatFilePos *dbp)
Store block on disk.
Definition: blockstorage.h:316
bool FlushChainstateBlockFile(int tip_height)
void FindFilesToPrune(std::set< int > &setFilesToPrune, int last_prune, const Chainstate &chain, ChainstateManager &chainman)
Prune block and undo files (blk???.dat and rev???.dat) so that the disk space used is less than a use...
FlatFileSeq BlockFileSeq() const
static constexpr auto PRUNE_TARGET_MANUAL
Definition: blockstorage.h:323
CBlockIndex * LookupBlockIndex(const uint256 &hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
std::array< std::optional< BlockfileCursor >, BlockfileType::NUM_TYPES > m_blockfile_cursors GUARDED_BY(cs_LastBlockFile)
Since assumedvalid chainstates may be syncing a range of the chain that is very far away from the nor...
std::unordered_map< std::string, PruneLockInfo > m_prune_locks GUARDED_BY(::cs_main)
Map from external index name to oldest block that must not be pruned.
void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
Mark one block file as pruned (modify associated database entries)
BlockfileType BlockfileTypeForHeight(int height)
std::vector< CBlockIndex * > GetAllBlockIndices() EXCLUSIVE_LOCKS_REQUIRED(std::multimap< CBlockIndex *, CBlockIndex * > m_blocks_unlinked
All pairs A->B, where A (or one of its ancestors) misses transactions, but B has transactions.
Definition: blockstorage.h:278
BlockManager(const util::SignalInterrupt &interrupt, Options opts)
Definition: blockstorage.h:254
bool ReadRawBlockFromDisk(std::vector< uint8_t > &block, const FlatFilePos &pos) const
std::set< CBlockIndex * > m_dirty_blockindex
Dirty block index entries.
Definition: blockstorage.h:234
bool LoadingBlocks() const
Definition: blockstorage.h:325
bool UndoReadFromDisk(CBlockUndo &blockundo, const CBlockIndex &index) const
fs::path GetBlockPosFilename(const FlatFilePos &pos) const
Translation to a filesystem path.
bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown)
bool FlushBlockFile(int blockfile_num, bool fFinalize, bool finalize_undo)
Return false if block file or undo file flushing fails.
uint64_t GetPruneTarget() const
Attempt to stay below this number of bytes of block files.
Definition: blockstorage.h:322
bool WriteBlockToDisk(const CBlock &block, FlatFilePos &pos) const
int MaxBlockfileNum() const EXCLUSIVE_LOCKS_REQUIRED(cs_LastBlockFile)
Definition: blockstorage.h:217
bool CheckBlockDataAvailability(const CBlockIndex &upper_block LIFETIMEBOUND, const CBlockIndex &lower_block LIFETIMEBOUND) EXCLUSIVE_LOCKS_REQUIRED(const CBlockIndex *GetFirstStoredBlock(const CBlockIndex &start_block LIFETIMEBOUND, const CBlockIndex *lower_block=nullptr) EXCLUSIVE_LOCKS_REQUIRED(boo m_have_pruned)
Check if all blocks in the [upper_block, lower_block] range have data available.
Definition: blockstorage.h:344
void UnlinkPrunedFiles(const std::set< int > &setFilesToPrune) const
Actually unlink the specified files.
bool WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(bool LoadBlockIndexDB(const std::optional< uint256 > &snapshot_blockhash) EXCLUSIVE_LOCKS_REQUIRED(void ScanAndUnlinkAlreadyPrunedFiles() EXCLUSIVE_LOCKS_REQUIRED(CBlockIndex * AddToBlockIndex(const CBlockHeader &block, CBlockIndex *&best_header) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
Remove any pruned block & undo files that are still on disk.
Definition: blockstorage.h:299
const bool m_prune_mode
Definition: blockstorage.h:231
bool FlushUndoFile(int block_file, bool finalize=false)
Return false if undo file flushing fails.
uint64_t CalculateCurrentUsage()
Calculate the amount of disk space the block & undo files currently use.
const util::SignalInterrupt & m_interrupt
Definition: blockstorage.h:257
const CBlockIndex * GetLastCheckpoint(const CCheckpointData &data) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
Returns last CBlockIndex* that is a checkpoint.
CBlockIndex * InsertBlockIndex(const uint256 &hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
Create a new block index entry for a given block hash.
bool m_check_for_pruning
Global flag to indicate we should check to see if there are block/undo files that should be deleted.
Definition: blockstorage.h:229
const CChainParams & GetParams() const
Definition: blockstorage.h:142
bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize)
bool IsBlockPruned(const CBlockIndex &block) EXCLUSIVE_LOCKS_REQUIRED(void UpdatePruneLock(const std::string &name, const PruneLockInfo &lock_info) EXCLUSIVE_LOCKS_REQUIRED(AutoFile OpenBlockFile(const FlatFilePos &pos, bool fReadOnly=false) const
Check whether the block associated with this index entry is pruned or not.
Definition: blockstorage.h:353
bool IsPruneMode() const
Whether running in -prune mode.
Definition: blockstorage.h:319
void CleanupBlockRevFiles() const
void FindFilesToPruneManual(std::set< int > &setFilesToPrune, int nManualPruneHeight, const Chainstate &chain, ChainstateManager &chainman)
std::atomic< bool > m_importing
Definition: blockstorage.h:260
std::vector< CBlockFileInfo > m_blockfile_info
Definition: blockstorage.h:200
bool UndoWriteToDisk(const CBlockUndo &blockundo, FlatFilePos &pos, const uint256 &hashBlock) const
CBlockFileInfo * GetBlockFileInfo(size_t n)
Get block file info entry for one block file.
bool LoadBlockIndex(const std::optional< uint256 > &snapshot_blockhash) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
Load the blocktree off disk and into memory.
AutoFile OpenUndoFile(const FlatFilePos &pos, bool fReadOnly=false) const
Open an undo file (rev?????.dat)
std::optional< int > m_snapshot_height
The height of the base block of an assumeutxo snapshot, if one is in use.
Definition: blockstorage.h:276
BlockMap m_block_index GUARDED_BY(cs_main)
256-bit opaque blob.
Definition: uint256.h:106
Helper class that manages an interrupt flag, and allows a thread or signal to interrupt another threa...
RecursiveMutex cs_main
Mutex to guard access to validation specific variables, such as reading or changing the chainstate.
Definition: cs_main.cpp:8
unsigned int nHeight
Transaction validation functions.
Filesystem operations and types.
Definition: init.h:25
static const unsigned int UNDOFILE_CHUNK_SIZE
The pre-allocation chunk size for rev?????.dat files (since 0.8)
Definition: blockstorage.h:72
BlockfileType
Definition: blockstorage.h:100
@ NUM_TYPES
Definition: blockstorage.h:104
@ NORMAL
Definition: blockstorage.h:102
@ ASSUMED
Definition: blockstorage.h:103
void ImportBlocks(ChainstateManager &chainman, std::vector< fs::path > vImportFiles)
static const unsigned int BLOCKFILE_CHUNK_SIZE
The pre-allocation chunk size for blk?????.dat files (since 0.8)
Definition: blockstorage.h:70
std::ostream & operator<<(std::ostream &os, const BlockfileType &type)
std::unordered_map< uint256, CBlockIndex, BlockHasher > BlockMap
Definition: blockstorage.h:85
static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE
Size of header written by WriteBlockToDisk before a serialized CBlock.
Definition: blockstorage.h:77
static const unsigned int MAX_BLOCKFILE_SIZE
The maximum size of a blk?????.dat file (since 0.8)
Definition: blockstorage.h:74
std::atomic_bool fReindex
const char * name
Definition: rest.cpp:49
Parameters that influence chain consensus.
Definition: params.h:74
An options struct for BlockManager, more ergonomically referred to as BlockManager::Options due to th...
const CChainParams & chainparams
bool operator()(const CBlockIndex *pa, const CBlockIndex *pb) const
bool operator()(const CBlockIndex *pa, const CBlockIndex *pb) const
#define EXCLUSIVE_LOCKS_REQUIRED(...)
Definition: threadsafety.h:49