Bitcoin Core  27.99.0
P2P Digital Currency
blockstorage.cpp
Go to the documentation of this file.
1 // Copyright (c) 2011-2022 The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 
5 #include <node/blockstorage.h>
6 
7 #include <arith_uint256.h>
8 #include <chain.h>
9 #include <consensus/params.h>
10 #include <consensus/validation.h>
11 #include <dbwrapper.h>
12 #include <flatfile.h>
13 #include <hash.h>
15 #include <kernel/chainparams.h>
18 #include <logging.h>
19 #include <pow.h>
20 #include <primitives/block.h>
21 #include <primitives/transaction.h>
22 #include <reverse_iterator.h>
23 #include <serialize.h>
24 #include <signet.h>
25 #include <span.h>
26 #include <streams.h>
27 #include <sync.h>
28 #include <tinyformat.h>
29 #include <uint256.h>
30 #include <undo.h>
31 #include <util/batchpriority.h>
32 #include <util/check.h>
33 #include <util/fs.h>
34 #include <util/signalinterrupt.h>
35 #include <util/strencodings.h>
36 #include <util/translation.h>
37 #include <validation.h>
38 
39 #include <map>
40 #include <unordered_map>
41 
42 namespace kernel {
43 static constexpr uint8_t DB_BLOCK_FILES{'f'};
44 static constexpr uint8_t DB_BLOCK_INDEX{'b'};
45 static constexpr uint8_t DB_FLAG{'F'};
46 static constexpr uint8_t DB_REINDEX_FLAG{'R'};
47 static constexpr uint8_t DB_LAST_BLOCK{'l'};
48 // Keys used in previous version that might still be found in the DB:
49 // BlockTreeDB::DB_TXINDEX_BLOCK{'T'};
50 // BlockTreeDB::DB_TXINDEX{'t'}
51 // BlockTreeDB::ReadFlag("txindex")
52 
54 {
55  return Read(std::make_pair(DB_BLOCK_FILES, nFile), info);
56 }
57 
58 bool BlockTreeDB::WriteReindexing(bool fReindexing)
59 {
60  if (fReindexing) {
61  return Write(DB_REINDEX_FLAG, uint8_t{'1'});
62  } else {
63  return Erase(DB_REINDEX_FLAG);
64  }
65 }
66 
67 void BlockTreeDB::ReadReindexing(bool& fReindexing)
68 {
69  fReindexing = Exists(DB_REINDEX_FLAG);
70 }
71 
73 {
74  return Read(DB_LAST_BLOCK, nFile);
75 }
76 
77 bool BlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*>>& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo)
78 {
79  CDBBatch batch(*this);
80  for (const auto& [file, info] : fileInfo) {
81  batch.Write(std::make_pair(DB_BLOCK_FILES, file), *info);
82  }
83  batch.Write(DB_LAST_BLOCK, nLastFile);
84  for (const CBlockIndex* bi : blockinfo) {
85  batch.Write(std::make_pair(DB_BLOCK_INDEX, bi->GetBlockHash()), CDiskBlockIndex{bi});
86  }
87  return WriteBatch(batch, true);
88 }
89 
90 bool BlockTreeDB::WriteFlag(const std::string& name, bool fValue)
91 {
92  return Write(std::make_pair(DB_FLAG, name), fValue ? uint8_t{'1'} : uint8_t{'0'});
93 }
94 
95 bool BlockTreeDB::ReadFlag(const std::string& name, bool& fValue)
96 {
97  uint8_t ch;
98  if (!Read(std::make_pair(DB_FLAG, name), ch)) {
99  return false;
100  }
101  fValue = ch == uint8_t{'1'};
102  return true;
103 }
104 
105 bool BlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex, const util::SignalInterrupt& interrupt)
106 {
108  std::unique_ptr<CDBIterator> pcursor(NewIterator());
109  pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256()));
110 
111  // Load m_block_index
112  while (pcursor->Valid()) {
113  if (interrupt) return false;
114  std::pair<uint8_t, uint256> key;
115  if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) {
116  CDiskBlockIndex diskindex;
117  if (pcursor->GetValue(diskindex)) {
118  // Construct block index object
119  CBlockIndex* pindexNew = insertBlockIndex(diskindex.ConstructBlockHash());
120  pindexNew->pprev = insertBlockIndex(diskindex.hashPrev);
121  pindexNew->nHeight = diskindex.nHeight;
122  pindexNew->nFile = diskindex.nFile;
123  pindexNew->nDataPos = diskindex.nDataPos;
124  pindexNew->nUndoPos = diskindex.nUndoPos;
125  pindexNew->nVersion = diskindex.nVersion;
126  pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot;
127  pindexNew->nTime = diskindex.nTime;
128  pindexNew->nBits = diskindex.nBits;
129  pindexNew->nNonce = diskindex.nNonce;
130  pindexNew->nStatus = diskindex.nStatus;
131  pindexNew->nTx = diskindex.nTx;
132 
133  if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, consensusParams)) {
134  LogError("%s: CheckProofOfWork failed: %s\n", __func__, pindexNew->ToString());
135  return false;
136  }
137 
138  pcursor->Next();
139  } else {
140  LogError("%s: failed to read value\n", __func__);
141  return false;
142  }
143  } else {
144  break;
145  }
146  }
147 
148  return true;
149 }
150 } // namespace kernel
151 
152 namespace node {
153 std::atomic_bool fReindex(false);
154 
156 {
157  // First sort by most total work, ...
158  if (pa->nChainWork > pb->nChainWork) return false;
159  if (pa->nChainWork < pb->nChainWork) return true;
160 
161  // ... then by earliest time received, ...
162  if (pa->nSequenceId < pb->nSequenceId) return false;
163  if (pa->nSequenceId > pb->nSequenceId) return true;
164 
165  // Use pointer address as tie breaker (should only happen with blocks
166  // loaded from disk, as those all have id 0).
167  if (pa < pb) return false;
168  if (pa > pb) return true;
169 
170  // Identical blocks.
171  return false;
172 }
173 
175 {
176  return pa->nHeight < pb->nHeight;
177 }
178 
179 std::vector<CBlockIndex*> BlockManager::GetAllBlockIndices()
180 {
182  std::vector<CBlockIndex*> rv;
183  rv.reserve(m_block_index.size());
184  for (auto& [_, block_index] : m_block_index) {
185  rv.push_back(&block_index);
186  }
187  return rv;
188 }
189 
191 {
193  BlockMap::iterator it = m_block_index.find(hash);
194  return it == m_block_index.end() ? nullptr : &it->second;
195 }
196 
198 {
200  BlockMap::const_iterator it = m_block_index.find(hash);
201  return it == m_block_index.end() ? nullptr : &it->second;
202 }
203 
205 {
207 
208  auto [mi, inserted] = m_block_index.try_emplace(block.GetHash(), block);
209  if (!inserted) {
210  return &mi->second;
211  }
212  CBlockIndex* pindexNew = &(*mi).second;
213 
214  // We assign the sequence id to blocks only when the full data is available,
215  // to avoid miners withholding blocks but broadcasting headers, to get a
216  // competitive advantage.
217  pindexNew->nSequenceId = 0;
218 
219  pindexNew->phashBlock = &((*mi).first);
220  BlockMap::iterator miPrev = m_block_index.find(block.hashPrevBlock);
221  if (miPrev != m_block_index.end()) {
222  pindexNew->pprev = &(*miPrev).second;
223  pindexNew->nHeight = pindexNew->pprev->nHeight + 1;
224  pindexNew->BuildSkip();
225  }
226  pindexNew->nTimeMax = (pindexNew->pprev ? std::max(pindexNew->pprev->nTimeMax, pindexNew->nTime) : pindexNew->nTime);
227  pindexNew->nChainWork = (pindexNew->pprev ? pindexNew->pprev->nChainWork : 0) + GetBlockProof(*pindexNew);
228  pindexNew->RaiseValidity(BLOCK_VALID_TREE);
229  if (best_header == nullptr || best_header->nChainWork < pindexNew->nChainWork) {
230  best_header = pindexNew;
231  }
232 
233  m_dirty_blockindex.insert(pindexNew);
234 
235  return pindexNew;
236 }
237 
238 void BlockManager::PruneOneBlockFile(const int fileNumber)
239 {
242 
243  for (auto& entry : m_block_index) {
244  CBlockIndex* pindex = &entry.second;
245  if (pindex->nFile == fileNumber) {
246  pindex->nStatus &= ~BLOCK_HAVE_DATA;
247  pindex->nStatus &= ~BLOCK_HAVE_UNDO;
248  pindex->nFile = 0;
249  pindex->nDataPos = 0;
250  pindex->nUndoPos = 0;
251  m_dirty_blockindex.insert(pindex);
252 
253  // Prune from m_blocks_unlinked -- any block we prune would have
254  // to be downloaded again in order to consider its chain, at which
255  // point it would be considered as a candidate for
256  // m_blocks_unlinked or setBlockIndexCandidates.
257  auto range = m_blocks_unlinked.equal_range(pindex->pprev);
258  while (range.first != range.second) {
259  std::multimap<CBlockIndex*, CBlockIndex*>::iterator _it = range.first;
260  range.first++;
261  if (_it->second == pindex) {
262  m_blocks_unlinked.erase(_it);
263  }
264  }
265  }
266  }
267 
268  m_blockfile_info.at(fileNumber) = CBlockFileInfo{};
269  m_dirty_fileinfo.insert(fileNumber);
270 }
271 
273  std::set<int>& setFilesToPrune,
274  int nManualPruneHeight,
275  const Chainstate& chain,
276  ChainstateManager& chainman)
277 {
278  assert(IsPruneMode() && nManualPruneHeight > 0);
279 
281  if (chain.m_chain.Height() < 0) {
282  return;
283  }
284 
285  const auto [min_block_to_prune, last_block_can_prune] = chainman.GetPruneRange(chain, nManualPruneHeight);
286 
287  int count = 0;
288  for (int fileNumber = 0; fileNumber < this->MaxBlockfileNum(); fileNumber++) {
289  const auto& fileinfo = m_blockfile_info[fileNumber];
290  if (fileinfo.nSize == 0 || fileinfo.nHeightLast > (unsigned)last_block_can_prune || fileinfo.nHeightFirst < (unsigned)min_block_to_prune) {
291  continue;
292  }
293 
294  PruneOneBlockFile(fileNumber);
295  setFilesToPrune.insert(fileNumber);
296  count++;
297  }
298  LogPrintf("[%s] Prune (Manual): prune_height=%d removed %d blk/rev pairs\n",
299  chain.GetRole(), last_block_can_prune, count);
300 }
301 
303  std::set<int>& setFilesToPrune,
304  int last_prune,
305  const Chainstate& chain,
306  ChainstateManager& chainman)
307 {
309  // Distribute our -prune budget over all chainstates.
310  const auto target = std::max(
311  MIN_DISK_SPACE_FOR_BLOCK_FILES, GetPruneTarget() / chainman.GetAll().size());
312  const uint64_t target_sync_height = chainman.m_best_header->nHeight;
313 
314  if (chain.m_chain.Height() < 0 || target == 0) {
315  return;
316  }
317  if (static_cast<uint64_t>(chain.m_chain.Height()) <= chainman.GetParams().PruneAfterHeight()) {
318  return;
319  }
320 
321  const auto [min_block_to_prune, last_block_can_prune] = chainman.GetPruneRange(chain, last_prune);
322 
323  uint64_t nCurrentUsage = CalculateCurrentUsage();
324  // We don't check to prune until after we've allocated new space for files
325  // So we should leave a buffer under our target to account for another allocation
326  // before the next pruning.
327  uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE;
328  uint64_t nBytesToPrune;
329  int count = 0;
330 
331  if (nCurrentUsage + nBuffer >= target) {
332  // On a prune event, the chainstate DB is flushed.
333  // To avoid excessive prune events negating the benefit of high dbcache
334  // values, we should not prune too rapidly.
335  // So when pruning in IBD, increase the buffer to avoid a re-prune too soon.
336  const auto chain_tip_height = chain.m_chain.Height();
337  if (chainman.IsInitialBlockDownload() && target_sync_height > (uint64_t)chain_tip_height) {
338  // Since this is only relevant during IBD, we assume blocks are at least 1 MB on average
339  static constexpr uint64_t average_block_size = 1000000; /* 1 MB */
340  const uint64_t remaining_blocks = target_sync_height - chain_tip_height;
341  nBuffer += average_block_size * remaining_blocks;
342  }
343 
344  for (int fileNumber = 0; fileNumber < this->MaxBlockfileNum(); fileNumber++) {
345  const auto& fileinfo = m_blockfile_info[fileNumber];
346  nBytesToPrune = fileinfo.nSize + fileinfo.nUndoSize;
347 
348  if (fileinfo.nSize == 0) {
349  continue;
350  }
351 
352  if (nCurrentUsage + nBuffer < target) { // are we below our target?
353  break;
354  }
355 
356  // don't prune files that could have a block that's not within the allowable
357  // prune range for the chain being pruned.
358  if (fileinfo.nHeightLast > (unsigned)last_block_can_prune || fileinfo.nHeightFirst < (unsigned)min_block_to_prune) {
359  continue;
360  }
361 
362  PruneOneBlockFile(fileNumber);
363  // Queue up the files for removal
364  setFilesToPrune.insert(fileNumber);
365  nCurrentUsage -= nBytesToPrune;
366  count++;
367  }
368  }
369 
370  LogPrint(BCLog::PRUNE, "[%s] target=%dMiB actual=%dMiB diff=%dMiB min_height=%d max_prune_height=%d removed %d blk/rev pairs\n",
371  chain.GetRole(), target / 1024 / 1024, nCurrentUsage / 1024 / 1024,
372  (int64_t(target) - int64_t(nCurrentUsage)) / 1024 / 1024,
373  min_block_to_prune, last_block_can_prune, count);
374 }
375 
376 void BlockManager::UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) {
378  m_prune_locks[name] = lock_info;
379 }
380 
382 {
384 
385  if (hash.IsNull()) {
386  return nullptr;
387  }
388 
389  const auto [mi, inserted]{m_block_index.try_emplace(hash)};
390  CBlockIndex* pindex = &(*mi).second;
391  if (inserted) {
392  pindex->phashBlock = &((*mi).first);
393  }
394  return pindex;
395 }
396 
397 bool BlockManager::LoadBlockIndex(const std::optional<uint256>& snapshot_blockhash)
398 {
399  if (!m_block_tree_db->LoadBlockIndexGuts(
400  GetConsensus(), [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); }, m_interrupt)) {
401  return false;
402  }
403 
404  if (snapshot_blockhash) {
405  const std::optional<AssumeutxoData> maybe_au_data = GetParams().AssumeutxoForBlockhash(*snapshot_blockhash);
406  if (!maybe_au_data) {
407  m_opts.notifications.fatalError(strprintf(_("Assumeutxo data not found for the given blockhash '%s'."), snapshot_blockhash->ToString()));
408  return false;
409  }
410  const AssumeutxoData& au_data = *Assert(maybe_au_data);
411  m_snapshot_height = au_data.height;
412  CBlockIndex* base{LookupBlockIndex(*snapshot_blockhash)};
413 
414  // Since nChainTx (responsible for estimated progress) isn't persisted
415  // to disk, we must bootstrap the value for assumedvalid chainstates
416  // from the hardcoded assumeutxo chainparams.
417  base->nChainTx = au_data.nChainTx;
418  LogPrintf("[snapshot] set nChainTx=%d for %s\n", au_data.nChainTx, snapshot_blockhash->ToString());
419  } else {
420  // If this isn't called with a snapshot blockhash, make sure the cached snapshot height
421  // is null. This is relevant during snapshot completion, when the blockman may be loaded
422  // with a height that then needs to be cleared after the snapshot is fully validated.
423  m_snapshot_height.reset();
424  }
425 
426  Assert(m_snapshot_height.has_value() == snapshot_blockhash.has_value());
427 
428  // Calculate nChainWork
429  std::vector<CBlockIndex*> vSortedByHeight{GetAllBlockIndices()};
430  std::sort(vSortedByHeight.begin(), vSortedByHeight.end(),
432 
433  CBlockIndex* previous_index{nullptr};
434  for (CBlockIndex* pindex : vSortedByHeight) {
435  if (m_interrupt) return false;
436  if (previous_index && pindex->nHeight > previous_index->nHeight + 1) {
437  LogError("%s: block index is non-contiguous, index of height %d missing\n", __func__, previous_index->nHeight + 1);
438  return false;
439  }
440  previous_index = pindex;
441  pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex);
442  pindex->nTimeMax = (pindex->pprev ? std::max(pindex->pprev->nTimeMax, pindex->nTime) : pindex->nTime);
443 
444  // We can link the chain of blocks for which we've received transactions at some point, or
445  // blocks that are assumed-valid on the basis of snapshot load (see
446  // PopulateAndValidateSnapshot()).
447  // Pruned nodes may have deleted the block.
448  if (pindex->nTx > 0) {
449  if (pindex->pprev) {
450  if (m_snapshot_height && pindex->nHeight == *m_snapshot_height &&
451  pindex->GetBlockHash() == *snapshot_blockhash) {
452  // Should have been set above; don't disturb it with code below.
453  Assert(pindex->nChainTx > 0);
454  } else if (pindex->pprev->nChainTx > 0) {
455  pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx;
456  } else {
457  pindex->nChainTx = 0;
458  m_blocks_unlinked.insert(std::make_pair(pindex->pprev, pindex));
459  }
460  } else {
461  pindex->nChainTx = pindex->nTx;
462  }
463  }
464  if (!(pindex->nStatus & BLOCK_FAILED_MASK) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_MASK)) {
465  pindex->nStatus |= BLOCK_FAILED_CHILD;
466  m_dirty_blockindex.insert(pindex);
467  }
468  if (pindex->pprev) {
469  pindex->BuildSkip();
470  }
471  }
472 
473  return true;
474 }
475 
476 bool BlockManager::WriteBlockIndexDB()
477 {
479  std::vector<std::pair<int, const CBlockFileInfo*>> vFiles;
480  vFiles.reserve(m_dirty_fileinfo.size());
481  for (std::set<int>::iterator it = m_dirty_fileinfo.begin(); it != m_dirty_fileinfo.end();) {
482  vFiles.emplace_back(*it, &m_blockfile_info[*it]);
483  m_dirty_fileinfo.erase(it++);
484  }
485  std::vector<const CBlockIndex*> vBlocks;
486  vBlocks.reserve(m_dirty_blockindex.size());
487  for (std::set<CBlockIndex*>::iterator it = m_dirty_blockindex.begin(); it != m_dirty_blockindex.end();) {
488  vBlocks.push_back(*it);
489  m_dirty_blockindex.erase(it++);
490  }
491  int max_blockfile = WITH_LOCK(cs_LastBlockFile, return this->MaxBlockfileNum());
492  if (!m_block_tree_db->WriteBatchSync(vFiles, max_blockfile, vBlocks)) {
493  return false;
494  }
495  return true;
496 }
497 
498 bool BlockManager::LoadBlockIndexDB(const std::optional<uint256>& snapshot_blockhash)
499 {
500  if (!LoadBlockIndex(snapshot_blockhash)) {
501  return false;
502  }
503  int max_blockfile_num{0};
504 
505  // Load block file info
506  m_block_tree_db->ReadLastBlockFile(max_blockfile_num);
507  m_blockfile_info.resize(max_blockfile_num + 1);
508  LogPrintf("%s: last block file = %i\n", __func__, max_blockfile_num);
509  for (int nFile = 0; nFile <= max_blockfile_num; nFile++) {
510  m_block_tree_db->ReadBlockFileInfo(nFile, m_blockfile_info[nFile]);
511  }
512  LogPrintf("%s: last block file info: %s\n", __func__, m_blockfile_info[max_blockfile_num].ToString());
513  for (int nFile = max_blockfile_num + 1; true; nFile++) {
514  CBlockFileInfo info;
515  if (m_block_tree_db->ReadBlockFileInfo(nFile, info)) {
516  m_blockfile_info.push_back(info);
517  } else {
518  break;
519  }
520  }
521 
522  // Check presence of blk files
523  LogPrintf("Checking all blk files are present...\n");
524  std::set<int> setBlkDataFiles;
525  for (const auto& [_, block_index] : m_block_index) {
526  if (block_index.nStatus & BLOCK_HAVE_DATA) {
527  setBlkDataFiles.insert(block_index.nFile);
528  }
529  }
530  for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) {
531  FlatFilePos pos(*it, 0);
532  if (OpenBlockFile(pos, true).IsNull()) {
533  return false;
534  }
535  }
536 
537  {
538  // Initialize the blockfile cursors.
540  for (size_t i = 0; i < m_blockfile_info.size(); ++i) {
541  const auto last_height_in_file = m_blockfile_info[i].nHeightLast;
542  m_blockfile_cursors[BlockfileTypeForHeight(last_height_in_file)] = {static_cast<int>(i), 0};
543  }
544  }
545 
546  // Check whether we have ever pruned block & undo files
547  m_block_tree_db->ReadFlag("prunedblockfiles", m_have_pruned);
548  if (m_have_pruned) {
549  LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n");
550  }
551 
552  // Check whether we need to continue reindexing
553  bool fReindexing = false;
554  m_block_tree_db->ReadReindexing(fReindexing);
555  if (fReindexing) fReindex = true;
556 
557  return true;
558 }
559 
560 void BlockManager::ScanAndUnlinkAlreadyPrunedFiles()
561 {
563  int max_blockfile = WITH_LOCK(cs_LastBlockFile, return this->MaxBlockfileNum());
564  if (!m_have_pruned) {
565  return;
566  }
567 
568  std::set<int> block_files_to_prune;
569  for (int file_number = 0; file_number < max_blockfile; file_number++) {
570  if (m_blockfile_info[file_number].nSize == 0) {
571  block_files_to_prune.insert(file_number);
572  }
573  }
574 
575  UnlinkPrunedFiles(block_files_to_prune);
576 }
577 
579 {
580  const MapCheckpoints& checkpoints = data.mapCheckpoints;
581 
582  for (const MapCheckpoints::value_type& i : reverse_iterate(checkpoints)) {
583  const uint256& hash = i.second;
584  const CBlockIndex* pindex = LookupBlockIndex(hash);
585  if (pindex) {
586  return pindex;
587  }
588  }
589  return nullptr;
590 }
591 
592 bool BlockManager::IsBlockPruned(const CBlockIndex& block)
593 {
595  return m_have_pruned && !(block.nStatus & BLOCK_HAVE_DATA) && (block.nTx > 0);
596 }
597 
598 const CBlockIndex* BlockManager::GetFirstStoredBlock(const CBlockIndex& upper_block, const CBlockIndex* lower_block)
599 {
601  const CBlockIndex* last_block = &upper_block;
602  assert(last_block->nStatus & BLOCK_HAVE_DATA); // 'upper_block' must have data
603  while (last_block->pprev && (last_block->pprev->nStatus & BLOCK_HAVE_DATA)) {
604  if (lower_block) {
605  // Return if we reached the lower_block
606  if (last_block == lower_block) return lower_block;
607  // if range was surpassed, means that 'lower_block' is not part of the 'upper_block' chain
608  // and so far this is not allowed.
609  assert(last_block->nHeight >= lower_block->nHeight);
610  }
611  last_block = last_block->pprev;
612  }
613  assert(last_block != nullptr);
614  return last_block;
615 }
616 
617 bool BlockManager::CheckBlockDataAvailability(const CBlockIndex& upper_block, const CBlockIndex& lower_block)
618 {
619  if (!(upper_block.nStatus & BLOCK_HAVE_DATA)) return false;
620  return GetFirstStoredBlock(upper_block, &lower_block) == &lower_block;
621 }
622 
623 // If we're using -prune with -reindex, then delete block files that will be ignored by the
624 // reindex. Since reindexing works by starting at block file 0 and looping until a blockfile
625 // is missing, do the same here to delete any later block files after a gap. Also delete all
626 // rev files since they'll be rewritten by the reindex anyway. This ensures that m_blockfile_info
627 // is in sync with what's actually on disk by the time we start downloading, so that pruning
628 // works correctly.
630 {
631  std::map<std::string, fs::path> mapBlockFiles;
632 
633  // Glob all blk?????.dat and rev?????.dat files from the blocks directory.
634  // Remove the rev files immediately and insert the blk file paths into an
635  // ordered map keyed by block file index.
636  LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n");
637  for (fs::directory_iterator it(m_opts.blocks_dir); it != fs::directory_iterator(); it++) {
638  const std::string path = fs::PathToString(it->path().filename());
639  if (fs::is_regular_file(*it) &&
640  path.length() == 12 &&
641  path.substr(8,4) == ".dat")
642  {
643  if (path.substr(0, 3) == "blk") {
644  mapBlockFiles[path.substr(3, 5)] = it->path();
645  } else if (path.substr(0, 3) == "rev") {
646  remove(it->path());
647  }
648  }
649  }
650 
651  // Remove all block files that aren't part of a contiguous set starting at
652  // zero by walking the ordered map (keys are block file indices) by
653  // keeping a separate counter. Once we hit a gap (or if 0 doesn't exist)
654  // start removing block files.
655  int nContigCounter = 0;
656  for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) {
657  if (LocaleIndependentAtoi<int>(item.first) == nContigCounter) {
658  nContigCounter++;
659  continue;
660  }
661  remove(item.second);
662  }
663 }
664 
666 {
668 
669  return &m_blockfile_info.at(n);
670 }
671 
672 bool BlockManager::UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const
673 {
674  // Open history file to append
675  AutoFile fileout{OpenUndoFile(pos)};
676  if (fileout.IsNull()) {
677  LogError("%s: OpenUndoFile failed\n", __func__);
678  return false;
679  }
680 
681  // Write index header
682  unsigned int nSize = GetSerializeSize(blockundo);
683  fileout << GetParams().MessageStart() << nSize;
684 
685  // Write undo data
686  long fileOutPos = ftell(fileout.Get());
687  if (fileOutPos < 0) {
688  LogError("%s: ftell failed\n", __func__);
689  return false;
690  }
691  pos.nPos = (unsigned int)fileOutPos;
692  fileout << blockundo;
693 
694  // calculate & write checksum
695  HashWriter hasher{};
696  hasher << hashBlock;
697  hasher << blockundo;
698  fileout << hasher.GetHash();
699 
700  return true;
701 }
702 
703 bool BlockManager::UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& index) const
704 {
705  const FlatFilePos pos{WITH_LOCK(::cs_main, return index.GetUndoPos())};
706 
707  if (pos.IsNull()) {
708  LogError("%s: no undo data available\n", __func__);
709  return false;
710  }
711 
712  // Open history file to read
713  AutoFile filein{OpenUndoFile(pos, true)};
714  if (filein.IsNull()) {
715  LogError("%s: OpenUndoFile failed\n", __func__);
716  return false;
717  }
718 
719  // Read block
720  uint256 hashChecksum;
721  HashVerifier verifier{filein}; // Use HashVerifier as reserializing may lose data, c.f. commit d342424301013ec47dc146a4beb49d5c9319d80a
722  try {
723  verifier << index.pprev->GetBlockHash();
724  verifier >> blockundo;
725  filein >> hashChecksum;
726  } catch (const std::exception& e) {
727  LogError("%s: Deserialize or I/O error - %s\n", __func__, e.what());
728  return false;
729  }
730 
731  // Verify checksum
732  if (hashChecksum != verifier.GetHash()) {
733  LogError("%s: Checksum mismatch\n", __func__);
734  return false;
735  }
736 
737  return true;
738 }
739 
740 bool BlockManager::FlushUndoFile(int block_file, bool finalize)
741 {
742  FlatFilePos undo_pos_old(block_file, m_blockfile_info[block_file].nUndoSize);
743  if (!UndoFileSeq().Flush(undo_pos_old, finalize)) {
744  m_opts.notifications.flushError(_("Flushing undo file to disk failed. This is likely the result of an I/O error."));
745  return false;
746  }
747  return true;
748 }
749 
750 bool BlockManager::FlushBlockFile(int blockfile_num, bool fFinalize, bool finalize_undo)
751 {
752  bool success = true;
754 
755  if (m_blockfile_info.size() < 1) {
756  // Return if we haven't loaded any blockfiles yet. This happens during
757  // chainstate init, when we call ChainstateManager::MaybeRebalanceCaches() (which
758  // then calls FlushStateToDisk()), resulting in a call to this function before we
759  // have populated `m_blockfile_info` via LoadBlockIndexDB().
760  return true;
761  }
762  assert(static_cast<int>(m_blockfile_info.size()) > blockfile_num);
763 
764  FlatFilePos block_pos_old(blockfile_num, m_blockfile_info[blockfile_num].nSize);
765  if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) {
766  m_opts.notifications.flushError(_("Flushing block file to disk failed. This is likely the result of an I/O error."));
767  success = false;
768  }
769  // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks,
770  // e.g. during IBD or a sync after a node going offline
771  if (!fFinalize || finalize_undo) {
772  if (!FlushUndoFile(blockfile_num, finalize_undo)) {
773  success = false;
774  }
775  }
776  return success;
777 }
778 
780 {
781  if (!m_snapshot_height) {
782  return BlockfileType::NORMAL;
783  }
784  return (height >= *m_snapshot_height) ? BlockfileType::ASSUMED : BlockfileType::NORMAL;
785 }
786 
788 {
790  auto& cursor = m_blockfile_cursors[BlockfileTypeForHeight(tip_height)];
791  // If the cursor does not exist, it means an assumeutxo snapshot is loaded,
792  // but no blocks past the snapshot height have been written yet, so there
793  // is no data associated with the chainstate, and it is safe not to flush.
794  if (cursor) {
795  return FlushBlockFile(cursor->file_num, /*fFinalize=*/false, /*finalize_undo=*/false);
796  }
797  // No need to log warnings in this case.
798  return true;
799 }
800 
802 {
804 
805  uint64_t retval = 0;
806  for (const CBlockFileInfo& file : m_blockfile_info) {
807  retval += file.nSize + file.nUndoSize;
808  }
809  return retval;
810 }
811 
812 void BlockManager::UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const
813 {
814  std::error_code ec;
815  for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) {
816  FlatFilePos pos(*it, 0);
817  const bool removed_blockfile{fs::remove(BlockFileSeq().FileName(pos), ec)};
818  const bool removed_undofile{fs::remove(UndoFileSeq().FileName(pos), ec)};
819  if (removed_blockfile || removed_undofile) {
820  LogPrint(BCLog::BLOCKSTORAGE, "Prune: %s deleted blk/rev (%05u)\n", __func__, *it);
821  }
822  }
823 }
824 
826 {
827  return FlatFileSeq(m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE);
828 }
829 
831 {
833 }
834 
835 AutoFile BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const
836 {
837  return AutoFile{BlockFileSeq().Open(pos, fReadOnly)};
838 }
839 
841 AutoFile BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const
842 {
843  return AutoFile{UndoFileSeq().Open(pos, fReadOnly)};
844 }
845 
847 {
848  return BlockFileSeq().FileName(pos);
849 }
850 
851 bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown)
852 {
854 
855  const BlockfileType chain_type = BlockfileTypeForHeight(nHeight);
856 
857  if (!m_blockfile_cursors[chain_type]) {
858  // If a snapshot is loaded during runtime, we may not have initialized this cursor yet.
859  assert(chain_type == BlockfileType::ASSUMED);
860  const auto new_cursor = BlockfileCursor{this->MaxBlockfileNum() + 1};
861  m_blockfile_cursors[chain_type] = new_cursor;
862  LogPrint(BCLog::BLOCKSTORAGE, "[%s] initializing blockfile cursor to %s\n", chain_type, new_cursor);
863  }
864  const int last_blockfile = m_blockfile_cursors[chain_type]->file_num;
865 
866  int nFile = fKnown ? pos.nFile : last_blockfile;
867  if (static_cast<int>(m_blockfile_info.size()) <= nFile) {
868  m_blockfile_info.resize(nFile + 1);
869  }
870 
871  bool finalize_undo = false;
872  if (!fKnown) {
873  unsigned int max_blockfile_size{MAX_BLOCKFILE_SIZE};
874  // Use smaller blockfiles in test-only -fastprune mode - but avoid
875  // the possibility of having a block not fit into the block file.
876  if (m_opts.fast_prune) {
877  max_blockfile_size = 0x10000; // 64kiB
878  if (nAddSize >= max_blockfile_size) {
879  // dynamically adjust the blockfile size to be larger than the added size
880  max_blockfile_size = nAddSize + 1;
881  }
882  }
883  assert(nAddSize < max_blockfile_size);
884 
885  while (m_blockfile_info[nFile].nSize + nAddSize >= max_blockfile_size) {
886  // when the undo file is keeping up with the block file, we want to flush it explicitly
887  // when it is lagging behind (more blocks arrive than are being connected), we let the
888  // undo block write case handle it
889  finalize_undo = (static_cast<int>(m_blockfile_info[nFile].nHeightLast) ==
890  Assert(m_blockfile_cursors[chain_type])->undo_height);
891 
892  // Try the next unclaimed blockfile number
893  nFile = this->MaxBlockfileNum() + 1;
894  // Set to increment MaxBlockfileNum() for next iteration
895  m_blockfile_cursors[chain_type] = BlockfileCursor{nFile};
896 
897  if (static_cast<int>(m_blockfile_info.size()) <= nFile) {
898  m_blockfile_info.resize(nFile + 1);
899  }
900  }
901  pos.nFile = nFile;
902  pos.nPos = m_blockfile_info[nFile].nSize;
903  }
904 
905  if (nFile != last_blockfile) {
906  if (!fKnown) {
907  LogPrint(BCLog::BLOCKSTORAGE, "Leaving block file %i: %s (onto %i) (height %i)\n",
908  last_blockfile, m_blockfile_info[last_blockfile].ToString(), nFile, nHeight);
909 
910  // Do not propagate the return code. The flush concerns a previous block
911  // and undo file that has already been written to. If a flush fails
912  // here, and we crash, there is no expected additional block data
913  // inconsistency arising from the flush failure here. However, the undo
914  // data may be inconsistent after a crash if the flush is called during
915  // a reindex. A flush error might also leave some of the data files
916  // untrimmed.
917  if (!FlushBlockFile(last_blockfile, !fKnown, finalize_undo)) {
919  "Failed to flush previous block file %05i (finalize=%i, finalize_undo=%i) before opening new block file %05i\n",
920  last_blockfile, !fKnown, finalize_undo, nFile);
921  }
922  }
923  // No undo data yet in the new file, so reset our undo-height tracking.
924  m_blockfile_cursors[chain_type] = BlockfileCursor{nFile};
925  }
926 
927  m_blockfile_info[nFile].AddBlock(nHeight, nTime);
928  if (fKnown) {
929  m_blockfile_info[nFile].nSize = std::max(pos.nPos + nAddSize, m_blockfile_info[nFile].nSize);
930  } else {
931  m_blockfile_info[nFile].nSize += nAddSize;
932  }
933 
934  if (!fKnown) {
935  bool out_of_space;
936  size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space);
937  if (out_of_space) {
938  m_opts.notifications.fatalError(_("Disk space is too low!"));
939  return false;
940  }
941  if (bytes_allocated != 0 && IsPruneMode()) {
942  m_check_for_pruning = true;
943  }
944  }
945 
946  m_dirty_fileinfo.insert(nFile);
947  return true;
948 }
949 
950 bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize)
951 {
952  pos.nFile = nFile;
953 
955 
956  pos.nPos = m_blockfile_info[nFile].nUndoSize;
957  m_blockfile_info[nFile].nUndoSize += nAddSize;
958  m_dirty_fileinfo.insert(nFile);
959 
960  bool out_of_space;
961  size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space);
962  if (out_of_space) {
963  return FatalError(m_opts.notifications, state, _("Disk space is too low!"));
964  }
965  if (bytes_allocated != 0 && IsPruneMode()) {
966  m_check_for_pruning = true;
967  }
968 
969  return true;
970 }
971 
972 bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const
973 {
974  // Open history file to append
975  AutoFile fileout{OpenBlockFile(pos)};
976  if (fileout.IsNull()) {
977  LogError("WriteBlockToDisk: OpenBlockFile failed\n");
978  return false;
979  }
980 
981  // Write index header
982  unsigned int nSize = GetSerializeSize(TX_WITH_WITNESS(block));
983  fileout << GetParams().MessageStart() << nSize;
984 
985  // Write block
986  long fileOutPos = ftell(fileout.Get());
987  if (fileOutPos < 0) {
988  LogError("WriteBlockToDisk: ftell failed\n");
989  return false;
990  }
991  pos.nPos = (unsigned int)fileOutPos;
992  fileout << TX_WITH_WITNESS(block);
993 
994  return true;
995 }
996 
997 bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
998 {
1000  const BlockfileType type = BlockfileTypeForHeight(block.nHeight);
1001  auto& cursor = *Assert(WITH_LOCK(cs_LastBlockFile, return m_blockfile_cursors[type]));
1002 
1003  // Write undo information to disk
1004  if (block.GetUndoPos().IsNull()) {
1005  FlatFilePos _pos;
1006  if (!FindUndoPos(state, block.nFile, _pos, ::GetSerializeSize(blockundo) + 40)) {
1007  LogError("ConnectBlock(): FindUndoPos failed\n");
1008  return false;
1009  }
1010  if (!UndoWriteToDisk(blockundo, _pos, block.pprev->GetBlockHash())) {
1011  return FatalError(m_opts.notifications, state, _("Failed to write undo data."));
1012  }
1013  // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order)
1014  // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height
1015  // in the block file info as below; note that this does not catch the case where the undo writes are keeping up
1016  // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in
1017  // the FindBlockPos function
1018  if (_pos.nFile < cursor.file_num && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[_pos.nFile].nHeightLast) {
1019  // Do not propagate the return code, a failed flush here should not
1020  // be an indication for a failed write. If it were propagated here,
1021  // the caller would assume the undo data not to be written, when in
1022  // fact it is. Note though, that a failed flush might leave the data
1023  // file untrimmed.
1024  if (!FlushUndoFile(_pos.nFile, true)) {
1025  LogPrintLevel(BCLog::BLOCKSTORAGE, BCLog::Level::Warning, "Failed to flush undo file %05i\n", _pos.nFile);
1026  }
1027  } else if (_pos.nFile == cursor.file_num && block.nHeight > cursor.undo_height) {
1028  cursor.undo_height = block.nHeight;
1029  }
1030  // update nUndoPos in block index
1031  block.nUndoPos = _pos.nPos;
1032  block.nStatus |= BLOCK_HAVE_UNDO;
1033  m_dirty_blockindex.insert(&block);
1034  }
1035 
1036  return true;
1037 }
1038 
1040 {
1041  block.SetNull();
1042 
1043  // Open history file to read
1044  AutoFile filein{OpenBlockFile(pos, true)};
1045  if (filein.IsNull()) {
1046  LogError("ReadBlockFromDisk: OpenBlockFile failed for %s\n", pos.ToString());
1047  return false;
1048  }
1049 
1050  // Read block
1051  try {
1052  filein >> TX_WITH_WITNESS(block);
1053  } catch (const std::exception& e) {
1054  LogError("%s: Deserialize or I/O error - %s at %s\n", __func__, e.what(), pos.ToString());
1055  return false;
1056  }
1057 
1058  // Check the header
1059  if (!CheckProofOfWork(block.GetHash(), block.nBits, GetConsensus())) {
1060  LogError("ReadBlockFromDisk: Errors in block header at %s\n", pos.ToString());
1061  return false;
1062  }
1063 
1064  // Signet only: check block solution
1065  if (GetConsensus().signet_blocks && !CheckSignetBlockSolution(block, GetConsensus())) {
1066  LogError("ReadBlockFromDisk: Errors in block solution at %s\n", pos.ToString());
1067  return false;
1068  }
1069 
1070  return true;
1071 }
1072 
1073 bool BlockManager::ReadBlockFromDisk(CBlock& block, const CBlockIndex& index) const
1074 {
1075  const FlatFilePos block_pos{WITH_LOCK(cs_main, return index.GetBlockPos())};
1076 
1077  if (!ReadBlockFromDisk(block, block_pos)) {
1078  return false;
1079  }
1080  if (block.GetHash() != index.GetBlockHash()) {
1081  LogError("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s\n",
1082  index.ToString(), block_pos.ToString());
1083  return false;
1084  }
1085  return true;
1086 }
1087 
1088 bool BlockManager::ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos) const
1089 {
1090  FlatFilePos hpos = pos;
1091  // If nPos is less than 8 the pos is null and we don't have the block data
1092  // Return early to prevent undefined behavior of unsigned int underflow
1093  if (hpos.nPos < 8) {
1094  LogError("%s: OpenBlockFile failed for %s\n", __func__, pos.ToString());
1095  return false;
1096  }
1097  hpos.nPos -= 8; // Seek back 8 bytes for meta header
1098  AutoFile filein{OpenBlockFile(hpos, true)};
1099  if (filein.IsNull()) {
1100  LogError("%s: OpenBlockFile failed for %s\n", __func__, pos.ToString());
1101  return false;
1102  }
1103 
1104  try {
1105  MessageStartChars blk_start;
1106  unsigned int blk_size;
1107 
1108  filein >> blk_start >> blk_size;
1109 
1110  if (blk_start != GetParams().MessageStart()) {
1111  LogError("%s: Block magic mismatch for %s: %s versus expected %s\n", __func__, pos.ToString(),
1112  HexStr(blk_start),
1113  HexStr(GetParams().MessageStart()));
1114  return false;
1115  }
1116 
1117  if (blk_size > MAX_SIZE) {
1118  LogError("%s: Block data is larger than maximum deserialization size for %s: %s versus %s\n", __func__, pos.ToString(),
1119  blk_size, MAX_SIZE);
1120  return false;
1121  }
1122 
1123  block.resize(blk_size); // Zeroing of memory is intentional here
1124  filein.read(MakeWritableByteSpan(block));
1125  } catch (const std::exception& e) {
1126  LogError("%s: Read from block file failed: %s for %s\n", __func__, e.what(), pos.ToString());
1127  return false;
1128  }
1129 
1130  return true;
1131 }
1132 
1134 {
1135  unsigned int nBlockSize = ::GetSerializeSize(TX_WITH_WITNESS(block));
1136  FlatFilePos blockPos;
1137  const auto position_known {dbp != nullptr};
1138  if (position_known) {
1139  blockPos = *dbp;
1140  } else {
1141  // when known, blockPos.nPos points at the offset of the block data in the blk file. that already accounts for
1142  // the serialization header present in the file (the 4 magic message start bytes + the 4 length bytes = 8 bytes = BLOCK_SERIALIZATION_HEADER_SIZE).
1143  // we add BLOCK_SERIALIZATION_HEADER_SIZE only for new blocks since they will have the serialization header added when written to disk.
1144  nBlockSize += static_cast<unsigned int>(BLOCK_SERIALIZATION_HEADER_SIZE);
1145  }
1146  if (!FindBlockPos(blockPos, nBlockSize, nHeight, block.GetBlockTime(), position_known)) {
1147  LogError("%s: FindBlockPos failed\n", __func__);
1148  return FlatFilePos();
1149  }
1150  if (!position_known) {
1151  if (!WriteBlockToDisk(block, blockPos)) {
1152  m_opts.notifications.fatalError(_("Failed to write block."));
1153  return FlatFilePos();
1154  }
1155  }
1156  return blockPos;
1157 }
1158 
1160 {
1161  std::atomic<bool>& m_importing;
1162 
1163 public:
1164  ImportingNow(std::atomic<bool>& importing) : m_importing{importing}
1165  {
1166  assert(m_importing == false);
1167  m_importing = true;
1168  }
1170  {
1171  assert(m_importing == true);
1172  m_importing = false;
1173  }
1174 };
1175 
1176 void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFiles)
1177 {
1179 
1180  {
1181  ImportingNow imp{chainman.m_blockman.m_importing};
1182 
1183  // -reindex
1184  if (fReindex) {
1185  int nFile = 0;
1186  // Map of disk positions for blocks with unknown parent (only used for reindex);
1187  // parent hash -> child disk position, multiple children can have the same parent.
1188  std::multimap<uint256, FlatFilePos> blocks_with_unknown_parent;
1189  while (true) {
1190  FlatFilePos pos(nFile, 0);
1191  if (!fs::exists(chainman.m_blockman.GetBlockPosFilename(pos))) {
1192  break; // No block files left to reindex
1193  }
1194  AutoFile file{chainman.m_blockman.OpenBlockFile(pos, true)};
1195  if (file.IsNull()) {
1196  break; // This error is logged in OpenBlockFile
1197  }
1198  LogPrintf("Reindexing block file blk%05u.dat...\n", (unsigned int)nFile);
1199  chainman.LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent);
1200  if (chainman.m_interrupt) {
1201  LogPrintf("Interrupt requested. Exit %s\n", __func__);
1202  return;
1203  }
1204  nFile++;
1205  }
1206  WITH_LOCK(::cs_main, chainman.m_blockman.m_block_tree_db->WriteReindexing(false));
1207  fReindex = false;
1208  LogPrintf("Reindexing finished\n");
1209  // To avoid ending up in a situation without genesis block, re-try initializing (no-op if reindexing worked):
1210  chainman.ActiveChainstate().LoadGenesisBlock();
1211  }
1212 
1213  // -loadblock=
1214  for (const fs::path& path : vImportFiles) {
1215  AutoFile file{fsbridge::fopen(path, "rb")};
1216  if (!file.IsNull()) {
1217  LogPrintf("Importing blocks file %s...\n", fs::PathToString(path));
1218  chainman.LoadExternalBlockFile(file);
1219  if (chainman.m_interrupt) {
1220  LogPrintf("Interrupt requested. Exit %s\n", __func__);
1221  return;
1222  }
1223  } else {
1224  LogPrintf("Warning: Could not open blocks file %s\n", fs::PathToString(path));
1225  }
1226  }
1227 
1228  // scan for better chains in the block chain database, that are not yet connected in the active best chain
1229 
1230  // We can't hold cs_main during ActivateBestChain even though we're accessing
1231  // the chainman unique_ptrs since ABC requires us not to be holding cs_main, so retrieve
1232  // the relevant pointers before the ABC call.
1233  for (Chainstate* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) {
1234  BlockValidationState state;
1235  if (!chainstate->ActivateBestChain(state, nullptr)) {
1236  chainman.GetNotifications().fatalError(strprintf(_("Failed to connect best block (%s)."), state.ToString()));
1237  return;
1238  }
1239  }
1240  } // End scope of ImportingNow
1241 }
1242 
1243 std::ostream& operator<<(std::ostream& os, const BlockfileType& type) {
1244  switch(type) {
1245  case BlockfileType::NORMAL: os << "normal"; break;
1246  case BlockfileType::ASSUMED: os << "assumed"; break;
1247  default: os.setstate(std::ios_base::failbit);
1248  }
1249  return os;
1250 }
1251 
1252 std::ostream& operator<<(std::ostream& os, const BlockfileCursor& cursor) {
1253  os << strprintf("BlockfileCursor(file_num=%d, undo_height=%d)", cursor.file_num, cursor.undo_height);
1254  return os;
1255 }
1256 } // namespace node
void ScheduleBatchPriority()
On platforms that support it, tell the kernel the calling thread is CPU-intensive and non-interactive...
arith_uint256 GetBlockProof(const CBlockIndex &block)
Definition: chain.cpp:131
@ BLOCK_VALID_TREE
All parent headers found, difficulty matches, timestamp >= median previous, checkpoint.
Definition: chain.h:97
@ BLOCK_HAVE_UNDO
undo data available in rev*.dat
Definition: chain.h:122
@ BLOCK_HAVE_DATA
full block available in blk*.dat
Definition: chain.h:121
@ BLOCK_FAILED_CHILD
descends from failed block
Definition: chain.h:126
@ BLOCK_FAILED_MASK
Definition: chain.h:127
#define Assert(val)
Identity function.
Definition: check.h:77
Non-refcounted RAII wrapper for FILE*.
Definition: streams.h:389
Nodes collect new transactions into a block, hash them into a hash tree, and scan through nonce value...
Definition: block.h:22
uint32_t nBits
Definition: block.h:29
int64_t GetBlockTime() const
Definition: block.h:61
uint256 hashPrevBlock
Definition: block.h:26
uint256 GetHash() const
Definition: block.cpp:11
Definition: block.h:69
void SetNull()
Definition: block.h:95
The block chain is a tree shaped structure starting with the genesis block at the root,...
Definition: chain.h:141
uint256 hashMerkleRoot
Definition: chain.h:189
std::string ToString() const
Definition: chain.cpp:15
CBlockIndex * pprev
pointer to the index of the predecessor of this block
Definition: chain.h:147
void BuildSkip()
Build the skiplist pointer for this entry.
Definition: chain.cpp:125
arith_uint256 nChainWork
(memory only) Total amount of work (expected number of hashes) in the chain up to and including this ...
Definition: chain.h:165
uint32_t nTime
Definition: chain.h:190
unsigned int nTimeMax
(memory only) Maximum nTime in the chain up to and including this block.
Definition: chain.h:198
int32_t nSequenceId
(memory only) Sequential id assigned to distinguish order in which blocks are received.
Definition: chain.h:195
uint32_t nNonce
Definition: chain.h:192
uint256 GetBlockHash() const
Definition: chain.h:244
FlatFilePos GetUndoPos() const EXCLUSIVE_LOCKS_REQUIRED(
Definition: chain.h:220
uint32_t nBits
Definition: chain.h:191
bool RaiseValidity(enum BlockStatus nUpTo) EXCLUSIVE_LOCKS_REQUIRED(
Raise the validity level of this block index entry.
Definition: chain.h:308
unsigned int nTx
Number of transactions in this block.
Definition: chain.h:170
int32_t nVersion
block header
Definition: chain.h:188
int nHeight
height of the entry in the chain. The genesis block has height 0
Definition: chain.h:153
FlatFilePos GetBlockPos() const EXCLUSIVE_LOCKS_REQUIRED(
Definition: chain.h:209
unsigned int nChainTx
(memory only) Number of transactions in the chain up to and including this block.
Definition: chain.h:177
const uint256 * phashBlock
pointer to the hash of the block, if any. Memory is owned by this CBlockIndex
Definition: chain.h:144
Undo information for a CBlock.
Definition: undo.h:63
int Height() const
Return the maximal height in the chain.
Definition: chain.h:463
std::optional< AssumeutxoData > AssumeutxoForBlockhash(const uint256 &blockhash) const
Definition: chainparams.h:126
const MessageStartChars & MessageStart() const
Definition: chainparams.h:94
uint64_t PruneAfterHeight() const
Definition: chainparams.h:104
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:73
void Write(const K &key, const V &value)
Definition: dbwrapper.h:99
bool WriteBatch(CDBBatch &batch, bool fSync=false)
Definition: dbwrapper.cpp:291
bool Read(const K &key, V &value) const
Definition: dbwrapper.h:221
CDBIterator * NewIterator()
Definition: dbwrapper.cpp:393
bool Erase(const K &key, bool fSync=false)
Definition: dbwrapper.h:266
bool Write(const K &key, const V &value, bool fSync=false)
Definition: dbwrapper.h:241
bool Exists(const K &key) const
Definition: dbwrapper.h:257
Used to marshal pointers into hashes for db storage.
Definition: chain.h:356
uint256 hashPrev
Definition: chain.h:366
uint256 ConstructBlockHash() const
Definition: chain.h:400
Chainstate stores and provides an API to update our local knowledge of the current best chain.
Definition: validation.h:491
CChain m_chain
The current chain of blockheaders we consult and build on.
Definition: validation.h:571
Provides an interface for creating and interacting with one or two chainstates: an IBD chainstate gen...
Definition: validation.h:850
SnapshotCompletionResult MaybeCompleteSnapshotValidation() EXCLUSIVE_LOCKS_REQUIRED(const CBlockIndex *GetSnapshotBaseBlock() const EXCLUSIVE_LOCKS_REQUIRED(Chainstate ActiveChainstate)() const
Once the background validation chainstate has reached the height which is the base of the UTXO snapsh...
Definition: validation.h:1068
const CChainParams & GetParams() const
Definition: validation.h:934
bool IsInitialBlockDownload() const
Check whether we are doing an initial block download (synchronizing from disk or network)
const util::SignalInterrupt & m_interrupt
Definition: validation.h:961
void LoadExternalBlockFile(AutoFile &file_in, FlatFilePos *dbp=nullptr, std::multimap< uint256, FlatFilePos > *blocks_with_unknown_parent=nullptr)
Import blocks from an external file.
Chainstate &InitializeChainstate(CTxMemPool *mempool) EXCLUSIVE_LOCKS_REQUIRED(std::vector< Chainstate * GetAll)()
Instantiate a new chainstate.
Definition: validation.h:1037
kernel::Notifications & GetNotifications() const
Definition: validation.h:939
node::BlockManager m_blockman
A single BlockManager instance is shared across each constructed chainstate to avoid duplicating bloc...
Definition: validation.h:966
FlatFileSeq represents a sequence of numbered files storing raw data.
Definition: flatfile.h:46
fs::path FileName(const FlatFilePos &pos) const
Get the name of the file at the given position.
Definition: flatfile.cpp:28
size_t Allocate(const FlatFilePos &pos, size_t add_size, bool &out_of_space)
Allocate additional space in a file after the given starting position.
Definition: flatfile.cpp:55
FILE * Open(const FlatFilePos &pos, bool read_only=false)
Open a handle to the file at the given position.
Definition: flatfile.cpp:33
Reads data from an underlying stream, while hashing the read data.
Definition: hash.h:151
A writer stream (for serialization) that computes a 256-bit hash.
Definition: hash.h:101
std::string ToString() const
Definition: validation.h:128
constexpr bool IsNull() const
Definition: uint256.h:42
Path class wrapper to block calls to the fs::path(std::string) implicit constructor and the fs::path:...
Definition: fs.h:33
bool ReadLastBlockFile(int &nFile)
bool ReadFlag(const std::string &name, bool &fValue)
bool ReadBlockFileInfo(int nFile, CBlockFileInfo &info)
bool WriteBatchSync(const std::vector< std::pair< int, const CBlockFileInfo * >> &fileInfo, int nLastFile, const std::vector< const CBlockIndex * > &blockinfo)
void ReadReindexing(bool &fReindexing)
bool WriteFlag(const std::string &name, bool fValue)
bool WriteReindexing(bool fReindexing)
virtual void fatalError(const bilingual_str &message)
The fatal error notification is sent to notify the user when an error occurs in kernel code that can'...
virtual void flushError(const bilingual_str &message)
The flush error notification is sent to notify the user that an error occurred while flushing block d...
const kernel::BlockManagerOpts m_opts
Definition: blockstorage.h:249
std::set< int > m_dirty_fileinfo
Dirty block file entries.
Definition: blockstorage.h:237
bool ReadBlockFromDisk(CBlock &block, const FlatFilePos &pos) const
Functions for disk access for blocks.
FlatFileSeq UndoFileSeq() const
RecursiveMutex cs_LastBlockFile
Definition: blockstorage.h:199
const Consensus::Params & GetConsensus() const
Definition: blockstorage.h:143
bool WriteUndoDataForBlock(const CBlockUndo &blockundo, BlockValidationState &state, CBlockIndex &block) EXCLUSIVE_LOCKS_REQUIRED(FlatFilePos SaveBlockToDisk(const CBlock &block, int nHeight, const FlatFilePos *dbp)
Store block on disk.
Definition: blockstorage.h:316
bool FlushChainstateBlockFile(int tip_height)
void FindFilesToPrune(std::set< int > &setFilesToPrune, int last_prune, const Chainstate &chain, ChainstateManager &chainman)
Prune block and undo files (blk???.dat and rev???.dat) so that the disk space used is less than a use...
FlatFileSeq BlockFileSeq() const
CBlockIndex * LookupBlockIndex(const uint256 &hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
Mark one block file as pruned (modify associated database entries)
BlockfileType BlockfileTypeForHeight(int height)
std::vector< CBlockIndex * > GetAllBlockIndices() EXCLUSIVE_LOCKS_REQUIRED(std::multimap< CBlockIndex *, CBlockIndex * > m_blocks_unlinked
All pairs A->B, where A (or one of its ancestors) misses transactions, but B has transactions.
Definition: blockstorage.h:278
bool ReadRawBlockFromDisk(std::vector< uint8_t > &block, const FlatFilePos &pos) const
std::set< CBlockIndex * > m_dirty_blockindex
Dirty block index entries.
Definition: blockstorage.h:234
bool UndoReadFromDisk(CBlockUndo &blockundo, const CBlockIndex &index) const
fs::path GetBlockPosFilename(const FlatFilePos &pos) const
Translation to a filesystem path.
bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown)
bool FlushBlockFile(int blockfile_num, bool fFinalize, bool finalize_undo)
Return false if block file or undo file flushing fails.
uint64_t GetPruneTarget() const
Attempt to stay below this number of bytes of block files.
Definition: blockstorage.h:322
bool WriteBlockToDisk(const CBlock &block, FlatFilePos &pos) const
int MaxBlockfileNum() const EXCLUSIVE_LOCKS_REQUIRED(cs_LastBlockFile)
Definition: blockstorage.h:217
bool CheckBlockDataAvailability(const CBlockIndex &upper_block LIFETIMEBOUND, const CBlockIndex &lower_block LIFETIMEBOUND) EXCLUSIVE_LOCKS_REQUIRED(const CBlockIndex *GetFirstStoredBlock(const CBlockIndex &start_block LIFETIMEBOUND, const CBlockIndex *lower_block=nullptr) EXCLUSIVE_LOCKS_REQUIRED(boo m_have_pruned)
Check if all blocks in the [upper_block, lower_block] range have data available.
Definition: blockstorage.h:344
void UnlinkPrunedFiles(const std::set< int > &setFilesToPrune) const
Actually unlink the specified files.
bool WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(bool LoadBlockIndexDB(const std::optional< uint256 > &snapshot_blockhash) EXCLUSIVE_LOCKS_REQUIRED(void ScanAndUnlinkAlreadyPrunedFiles() EXCLUSIVE_LOCKS_REQUIRED(CBlockIndex * AddToBlockIndex(const CBlockHeader &block, CBlockIndex *&best_header) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
Remove any pruned block & undo files that are still on disk.
Definition: blockstorage.h:299
bool FlushUndoFile(int block_file, bool finalize=false)
Return false if undo file flushing fails.
uint64_t CalculateCurrentUsage()
Calculate the amount of disk space the block & undo files currently use.
const util::SignalInterrupt & m_interrupt
Definition: blockstorage.h:257
const CBlockIndex * GetLastCheckpoint(const CCheckpointData &data) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
Returns last CBlockIndex* that is a checkpoint.
CBlockIndex * InsertBlockIndex(const uint256 &hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
Create a new block index entry for a given block hash.
bool m_check_for_pruning
Global flag to indicate we should check to see if there are block/undo files that should be deleted.
Definition: blockstorage.h:229
const CChainParams & GetParams() const
Definition: blockstorage.h:142
bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize)
bool IsBlockPruned(const CBlockIndex &block) EXCLUSIVE_LOCKS_REQUIRED(void UpdatePruneLock(const std::string &name, const PruneLockInfo &lock_info) EXCLUSIVE_LOCKS_REQUIRED(AutoFile OpenBlockFile(const FlatFilePos &pos, bool fReadOnly=false) const
Check whether the block associated with this index entry is pruned or not.
Definition: blockstorage.h:353
bool IsPruneMode() const
Whether running in -prune mode.
Definition: blockstorage.h:319
void CleanupBlockRevFiles() const
void FindFilesToPruneManual(std::set< int > &setFilesToPrune, int nManualPruneHeight, const Chainstate &chain, ChainstateManager &chainman)
std::atomic< bool > m_importing
Definition: blockstorage.h:260
std::vector< CBlockFileInfo > m_blockfile_info
Definition: blockstorage.h:200
bool UndoWriteToDisk(const CBlockUndo &blockundo, FlatFilePos &pos, const uint256 &hashBlock) const
CBlockFileInfo * GetBlockFileInfo(size_t n)
Get block file info entry for one block file.
bool LoadBlockIndex(const std::optional< uint256 > &snapshot_blockhash) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
Load the blocktree off disk and into memory.
AutoFile OpenUndoFile(const FlatFilePos &pos, bool fReadOnly=false) const
Open an undo file (rev?????.dat)
std::optional< int > m_snapshot_height
The height of the base block of an assumeutxo snapshot, if one is in use.
Definition: blockstorage.h:276
ImportingNow(std::atomic< bool > &importing)
std::atomic< bool > & m_importing
256-bit opaque blob.
Definition: uint256.h:106
Helper class that manages an interrupt flag, and allows a thread or signal to interrupt another threa...
RecursiveMutex cs_main
Mutex to guard access to validation specific variables, such as reading or changing the chainstate.
Definition: cs_main.cpp:8
std::map< int, uint256 > MapCheckpoints
Definition: chainparams.h:27
#define LogPrintLevel(category, level,...)
Definition: logging.h:252
#define LogPrint(category,...)
Definition: logging.h:264
#define LogError(...)
Definition: logging.h:242
#define LogPrintf(...)
Definition: logging.h:245
unsigned int nHeight
std::array< uint8_t, 4 > MessageStartChars
@ BLOCKSTORAGE
Definition: logging.h:69
@ PRUNE
Definition: logging.h:55
static bool exists(const path &p)
Definition: fs.h:89
static std::string PathToString(const path &path)
Convert path object to a byte string.
Definition: fs.h:151
FILE * fopen(const fs::path &p, const char *mode)
Definition: fs.cpp:26
static constexpr uint8_t DB_REINDEX_FLAG
static constexpr uint8_t DB_FLAG
static constexpr uint8_t DB_BLOCK_INDEX
static constexpr uint8_t DB_LAST_BLOCK
static constexpr uint8_t DB_BLOCK_FILES
Definition: init.h:25
static const unsigned int UNDOFILE_CHUNK_SIZE
The pre-allocation chunk size for rev?????.dat files (since 0.8)
Definition: blockstorage.h:72
BlockfileType
Definition: blockstorage.h:100
@ ASSUMED
Definition: blockstorage.h:103
void ImportBlocks(ChainstateManager &chainman, std::vector< fs::path > vImportFiles)
static const unsigned int BLOCKFILE_CHUNK_SIZE
The pre-allocation chunk size for blk?????.dat files (since 0.8)
Definition: blockstorage.h:70
std::ostream & operator<<(std::ostream &os, const BlockfileType &type)
static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE
Size of header written by WriteBlockToDisk before a serialized CBlock.
Definition: blockstorage.h:77
static const unsigned int MAX_BLOCKFILE_SIZE
The maximum size of a blk?????.dat file (since 0.8)
Definition: blockstorage.h:74
std::atomic_bool fReindex
bool CheckProofOfWork(uint256 hash, unsigned int nBits, const Consensus::Params &params)
Check whether a block hash satisfies the proof-of-work requirement specified by nBits.
Definition: pow.cpp:125
static constexpr TransactionSerParams TX_WITH_WITNESS
Definition: transaction.h:195
const char * name
Definition: rest.cpp:50
reverse_range< T > reverse_iterate(T &x)
size_t GetSerializeSize(const T &t)
Definition: serialize.h:1116
static constexpr uint64_t MAX_SIZE
The maximum size of a serialized object in bytes or number of elements (for eg vectors) when the size...
Definition: serialize.h:32
bool CheckSignetBlockSolution(const CBlock &block, const Consensus::Params &consensusParams)
Extract signature and check whether a block has a valid solution.
Definition: signet.cpp:124
Span< std::byte > MakeWritableByteSpan(V &&v) noexcept
Definition: span.h:282
std::string ToString(const T &t)
Locale-independent version of std::to_string.
Definition: string.h:109
Holds configuration for use during UTXO snapshot load and validation.
Definition: chainparams.h:47
unsigned int nChainTx
Used to populate the nChainTx value, which is used during BlockManager::LoadBlockIndex().
Definition: chainparams.h:57
MapCheckpoints mapCheckpoints
Definition: chainparams.h:30
Parameters that influence chain consensus.
Definition: params.h:74
int nFile
Definition: flatfile.h:16
std::string ToString() const
Definition: flatfile.cpp:23
unsigned int nPos
Definition: flatfile.h:17
bool IsNull() const
Definition: flatfile.h:36
Notifications & notifications
bool operator()(const CBlockIndex *pa, const CBlockIndex *pb) const
bool operator()(const CBlockIndex *pa, const CBlockIndex *pb) const
#define LOCK2(cs1, cs2)
Definition: sync.h:258
#define LOCK(cs)
Definition: sync.h:257
#define WITH_LOCK(cs, code)
Run code while locking a mutex.
Definition: sync.h:301
static int count
#define EXCLUSIVE_LOCKS_REQUIRED(...)
Definition: threadsafety.h:49
#define strprintf
Format arguments and return the string or write to given std::ostream (see tinyformat::format doc for...
Definition: tinyformat.h:1162
bilingual_str _(const char *psz)
Translation function.
Definition: translation.h:74
std::string HexStr(const Span< const uint8_t > s)
Convert a span of bytes to a lower-case hexadecimal string.
bool FatalError(Notifications &notifications, BlockValidationState &state, const bilingual_str &message)
AssertLockHeld(pool.cs)
assert(!tx.IsCoinBase())
static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES
Definition: validation.h:77