Bitcoin ABC  0.24.7
P2P Digital Currency
txindex.cpp
Go to the documentation of this file.
1 // Copyright (c) 2017-2018 The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 
5 #include <index/txindex.h>
6 
7 #include <blockdb.h>
8 #include <chain.h>
9 #include <index/disktxpos.h>
10 #include <node/ui_interface.h>
11 #include <shutdown.h>
12 #include <util/system.h>
13 #include <util/translation.h>
14 #include <validation.h>
15 
16 constexpr char DB_BEST_BLOCK = 'B';
17 constexpr char DB_TXINDEX = 't';
18 constexpr char DB_TXINDEX_BLOCK = 'T';
19 
20 std::unique_ptr<TxIndex> g_txindex;
21 
23 class TxIndex::DB : public BaseIndex::DB {
24 public:
25  explicit DB(size_t n_cache_size, bool f_memory = false,
26  bool f_wipe = false);
27 
30  bool ReadTxPos(const TxId &txid, CDiskTxPos &pos) const;
31 
33  bool WriteTxs(const std::vector<std::pair<TxId, CDiskTxPos>> &v_pos);
34 
37  bool MigrateData(CBlockTreeDB &block_tree_db,
38  const CBlockLocator &best_locator);
39 };
40 
41 TxIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe)
42  : BaseIndex::DB(GetDataDir() / "indexes" / "txindex", n_cache_size,
43  f_memory, f_wipe) {}
44 
45 bool TxIndex::DB::ReadTxPos(const TxId &txid, CDiskTxPos &pos) const {
46  return Read(std::make_pair(DB_TXINDEX, txid), pos);
47 }
48 
50  const std::vector<std::pair<TxId, CDiskTxPos>> &v_pos) {
51  CDBBatch batch(*this);
52  for (const auto &tuple : v_pos) {
53  batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second);
54  }
55  return WriteBatch(batch);
56 }
57 
58 /*
59  * Safely persist a transfer of data from the old txindex database to the new
60  * one, and compact the range of keys updated. This is used internally by
61  * MigrateData.
62  */
63 static void
65  CDBBatch &batch_newdb, CDBBatch &batch_olddb,
66  const std::pair<uint8_t, TxId> &begin_key,
67  const std::pair<uint8_t, TxId> &end_key) {
68  // Sync new DB changes to disk before deleting from old DB.
69  newdb.WriteBatch(batch_newdb, /*fSync=*/true);
70  olddb.WriteBatch(batch_olddb);
71  olddb.CompactRange(begin_key, end_key);
72 
73  batch_newdb.Clear();
74  batch_olddb.Clear();
75 }
76 
78  const CBlockLocator &best_locator) {
79  // The prior implementation of txindex was always in sync with block index
80  // and presence was indicated with a boolean DB flag. If the flag is set,
81  // this means the txindex from a previous version is valid and in sync with
82  // the chain tip. The first step of the migration is to unset the flag and
83  // write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the
84  // index entries are copied over in batches to the new database. Finally,
85  // DB_TXINDEX_BLOCK is erased from the old database and the block hash is
86  // written to the new database.
87  //
88  // Unsetting the boolean flag ensures that if the node is downgraded to a
89  // previous version, it will not see a corrupted, partially migrated index
90  // -- it will see that the txindex is disabled. When the node is upgraded
91  // again, the migration will pick up where it left off and sync to the block
92  // with hash DB_TXINDEX_BLOCK.
93  bool f_legacy_flag = false;
94  block_tree_db.ReadFlag("txindex", f_legacy_flag);
95  if (f_legacy_flag) {
96  if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) {
97  return error("%s: cannot write block indicator", __func__);
98  }
99  if (!block_tree_db.WriteFlag("txindex", false)) {
100  return error("%s: cannot write block index db flag", __func__);
101  }
102  }
103 
104  CBlockLocator locator;
105  if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) {
106  return true;
107  }
108 
109  int64_t count = 0;
110  uiInterface.InitMessage(_("Upgrading txindex database").translated);
111  LogPrintf("Upgrading txindex database... [0%%]\n");
112  uiInterface.ShowProgress(_("Upgrading txindex database").translated, 0,
113  true);
114  int report_done = 0;
115  const size_t batch_size = 1 << 24; // 16 MiB
116 
117  CDBBatch batch_newdb(*this);
118  CDBBatch batch_olddb(block_tree_db);
119 
120  std::pair<uint8_t, TxId> key;
121  std::pair<uint8_t, TxId> begin_key{DB_TXINDEX, TxId()};
122  std::pair<uint8_t, TxId> prev_key = begin_key;
123 
124  bool interrupted = false;
125  std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator());
126  for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) {
127  if (ShutdownRequested()) {
128  interrupted = true;
129  break;
130  }
131 
132  if (!cursor->GetKey(key)) {
133  return error("%s: cannot get key from valid cursor", __func__);
134  }
135  if (key.first != DB_TXINDEX) {
136  break;
137  }
138 
139  // Log progress every 10%.
140  if (++count % 256 == 0) {
141  // Since txids are uniformly random and traversed in increasing
142  // order, the high 16 bits of the ID can be used to estimate the
143  // current progress.
144  const TxId &txid = key.second;
145  uint32_t high_nibble =
146  (static_cast<uint32_t>(*(txid.begin() + 0)) << 8) +
147  (static_cast<uint32_t>(*(txid.begin() + 1)) << 0);
148  int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5);
149 
150  uiInterface.ShowProgress(_("Upgrading txindex database").translated,
151  percentage_done, true);
152  if (report_done < percentage_done / 10) {
153  LogPrintf("Upgrading txindex database... [%d%%]\n",
154  percentage_done);
155  report_done = percentage_done / 10;
156  }
157  }
158 
159  CDiskTxPos value;
160  if (!cursor->GetValue(value)) {
161  return error("%s: cannot parse txindex record", __func__);
162  }
163  batch_newdb.Write(key, value);
164  batch_olddb.Erase(key);
165 
166  if (batch_newdb.SizeEstimate() > batch_size ||
167  batch_olddb.SizeEstimate() > batch_size) {
168  // NOTE: it's OK to delete the key pointed at by the current DB
169  // cursor while iterating because LevelDB iterators are guaranteed
170  // to provide a consistent view of the underlying data, like a
171  // lightweight snapshot.
172  WriteTxIndexMigrationBatches(*this, block_tree_db, batch_newdb,
173  batch_olddb, prev_key, key);
174  prev_key = key;
175  }
176  }
177 
178  // If these final DB batches complete the migration, write the best block
179  // hash marker to the new database and delete from the old one. This signals
180  // that the former is fully caught up to that point in the blockchain and
181  // that all txindex entries have been removed from the latter.
182  if (!interrupted) {
183  batch_olddb.Erase(DB_TXINDEX_BLOCK);
184  batch_newdb.Write(DB_BEST_BLOCK, locator);
185  }
186 
187  WriteTxIndexMigrationBatches(*this, block_tree_db, batch_newdb, batch_olddb,
188  begin_key, key);
189 
190  if (interrupted) {
191  LogPrintf("[CANCELLED].\n");
192  return false;
193  }
194 
195  uiInterface.ShowProgress("", 100, false);
196 
197  LogPrintf("[DONE].\n");
198  return true;
199 }
200 
201 TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe)
202  : m_db(std::make_unique<TxIndex::DB>(n_cache_size, f_memory, f_wipe)) {}
203 
205 
207  LOCK(cs_main);
208 
209  // Attempt to migrate txindex from the old database to the new one. Even if
210  // chain_tip is null, the node could be reindexing and we still want to
211  // delete txindex records in the old database.
212  if (!m_db->MigrateData(*pblocktree, ::ChainActive().GetLocator())) {
213  return false;
214  }
215 
216  return BaseIndex::Init();
217 }
218 
219 bool TxIndex::WriteBlock(const CBlock &block, const CBlockIndex *pindex) {
220  // Exclude genesis block transaction because outputs are not spendable.
221  if (pindex->nHeight == 0) {
222  return true;
223  }
224 
225  CDiskTxPos pos(pindex->GetBlockPos(),
226  GetSizeOfCompactSize(block.vtx.size()));
227  std::vector<std::pair<TxId, CDiskTxPos>> vPos;
228  vPos.reserve(block.vtx.size());
229  for (const auto &tx : block.vtx) {
230  vPos.emplace_back(tx->GetId(), pos);
232  }
233  return m_db->WriteTxs(vPos);
234 }
235 
237  return *m_db;
238 }
239 
240 bool TxIndex::FindTx(const TxId &txid, BlockHash &block_hash,
241  CTransactionRef &tx) const {
242  CDiskTxPos postx;
243  if (!m_db->ReadTxPos(txid, postx)) {
244  return false;
245  }
246 
247  CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION);
248  if (file.IsNull()) {
249  return error("%s: OpenBlockFile failed", __func__);
250  }
251  CBlockHeader header;
252  try {
253  file >> header;
254  if (fseek(file.Get(), postx.nTxOffset, SEEK_CUR)) {
255  return error("%s: fseek(...) failed", __func__);
256  }
257  file >> tx;
258  } catch (const std::exception &e) {
259  return error("%s: Deserialize or I/O error - %s", __func__, e.what());
260  }
261  if (tx->GetId() != txid) {
262  return error("%s: txid mismatch", __func__);
263  }
264  block_hash = header.GetHash();
265  return true;
266 }
GetSerializeSize
size_t GetSerializeSize(const T &t, int nVersion=0)
Definition: serialize.h:1182
OpenBlockFile
FILE * OpenBlockFile(const FlatFilePos &pos, bool fReadOnly)
Open a block file (blk?????.dat).
Definition: blockdb.cpp:20
ShutdownRequested
bool ShutdownRequested()
Definition: shutdown.cpp:18
TxIndex::DB
Access to the txindex database (indexes/txindex/)
Definition: txindex.cpp:23
TxIndex::Init
bool Init() override
Override base class init to migrate from old database.
Definition: txindex.cpp:206
_
bilingual_str _(const char *psz)
Translation function.
Definition: translation.h:55
GetDataDir
const fs::path & GetDataDir(bool fNetSpecific)
Definition: system.cpp:779
count
static int count
Definition: tests.c:41
TxIndex::DB::ReadTxPos
bool ReadTxPos(const TxId &txid, CDiskTxPos &pos) const
Read the disk location of the transaction data with the given ID.
Definition: txindex.cpp:45
g_txindex
std::unique_ptr< TxIndex > g_txindex
The global transaction index, used in GetTransaction. May be null.
Definition: txindex.cpp:20
TxIndex::WriteBlock
bool WriteBlock(const CBlock &block, const CBlockIndex *pindex) override
Write update index entries for a newly connected block.
Definition: txindex.cpp:219
ChainActive
CChain & ChainActive()
Please prefer the identical ChainstateManager::ActiveChain.
Definition: validation.cpp:86
CBlockHeader
Nodes collect new transactions into a block, hash them into a hash tree, and scan through nonce value...
Definition: block.h:22
CDBBatch
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:48
BaseIndex::Init
virtual bool Init()
Initialize internal state from the database and block index.
Definition: base.cpp:54
uiInterface
CClientUIInterface uiInterface
Definition: ui_interface.cpp:12
CAutoFile::Get
FILE * Get() const
Get wrapped FILE* without transfer of ownership.
Definition: streams.h:624
WriteTxIndexMigrationBatches
static void WriteTxIndexMigrationBatches(CDBWrapper &newdb, CDBWrapper &olddb, CDBBatch &batch_newdb, CDBBatch &batch_olddb, const std::pair< uint8_t, TxId > &begin_key, const std::pair< uint8_t, TxId > &end_key)
Definition: txindex.cpp:64
CBlockIndex::nHeight
int nHeight
height of the entry in the chain. The genesis block has height 0
Definition: blockindex.h:36
TxIndex::m_db
const std::unique_ptr< DB > m_db
Definition: txindex.h:20
CDiskTxPos
Definition: disktxpos.h:11
CDiskTxPos::nTxOffset
unsigned int nTxOffset
Definition: disktxpos.h:12
CDBWrapper::Read
bool Read(const K &key, V &value) const
Definition: dbwrapper.h:230
TxIndex::DB::WriteTxs
bool WriteTxs(const std::vector< std::pair< TxId, CDiskTxPos >> &v_pos)
Write a batch of transaction positions to the DB.
Definition: txindex.cpp:49
DB_TXINDEX_BLOCK
constexpr char DB_TXINDEX_BLOCK
Definition: txindex.cpp:18
BaseIndex
Base class for indices of blockchain data.
Definition: base.h:27
CBlockIndex::GetBlockPos
FlatFilePos GetBlockPos() const
Definition: blockindex.h:102
disktxpos.h
shutdown.h
txindex.h
CDBBatch::Write
void Write(const K &key, const V &value)
Definition: dbwrapper.h:73
CAutoFile
Non-refcounted RAII wrapper for FILE*.
Definition: streams.h:581
TxIndex::~TxIndex
virtual ~TxIndex() override
Definition: txindex.cpp:204
blockdb.h
CDBWrapper::NewIterator
CDBIterator * NewIterator()
Definition: dbwrapper.h:289
GetSizeOfCompactSize
uint32_t GetSizeOfCompactSize(uint64_t nSize)
Compact Size size < 253 – 1 byte size <= USHRT_MAX – 3 bytes (253 + 2 bytes) size <= UINT_MAX – 5 byt...
Definition: serialize.h:390
cs_main
RecursiveMutex cs_main
Global state.
Definition: validation.cpp:103
TxIndex::FindTx
bool FindTx(const TxId &txid, BlockHash &block_hash, CTransactionRef &tx) const
Look up a transaction by identifier.
Definition: txindex.cpp:240
DB_TXINDEX
constexpr char DB_TXINDEX
Definition: txindex.cpp:17
DB_BEST_BLOCK
constexpr char DB_BEST_BLOCK
Definition: txindex.cpp:16
CDBBatch::Clear
void Clear()
Definition: dbwrapper.h:68
CBlockTreeDB
Access to the block database (blocks/index/)
Definition: txdb.h:105
TxIndex::DB::MigrateData
bool MigrateData(CBlockTreeDB &block_tree_db, const CBlockLocator &best_locator)
Migrate txindex data from the block tree DB, where it may be for older nodes that have not been upgra...
Definition: txindex.cpp:77
CAutoFile::IsNull
bool IsNull() const
Return true if the wrapped FILE* is nullptr, false otherwise.
Definition: streams.h:627
bilingual_str::translated
std::string translated
Definition: translation.h:19
TxId
A TxId is the identifier of a transaction.
Definition: txid.h:14
CBlockTreeDB::ReadFlag
bool ReadFlag(const std::string &name, bool &fValue)
Definition: txdb.cpp:261
base_blob::begin
uint8_t * begin()
Definition: uint256.h:83
TxIndex::GetDB
BaseIndex::DB & GetDB() const override
Definition: txindex.cpp:236
BlockHash
A BlockHash is a unqiue identifier for a block.
Definition: blockhash.h:13
ui_interface.h
CDBWrapper
Definition: dbwrapper.h:171
system.h
CBlock
Definition: block.h:55
CLIENT_VERSION
static constexpr int CLIENT_VERSION
bitcoind-res.rc includes this file, but it cannot cope with real c++ code.
Definition: clientversion.h:44
CBlockTreeDB::WriteFlag
bool WriteFlag(const std::string &name, bool fValue)
Definition: txdb.cpp:257
CBlock::vtx
std::vector< CTransactionRef > vtx
Definition: block.h:58
translation.h
pblocktree
std::unique_ptr< CBlockTreeDB > pblocktree
Global variable that points to the active block tree (protected by cs_main)
Definition: validation.cpp:176
LOCK
#define LOCK(cs)
Definition: sync.h:241
TxIndex::DB::DB
DB(size_t n_cache_size, bool f_memory=false, bool f_wipe=false)
Definition: txindex.cpp:41
CDBBatch::Erase
void Erase(const K &key)
Definition: dbwrapper.h:97
CTransactionRef
std::shared_ptr< const CTransaction > CTransactionRef
Definition: transaction.h:319
TxIndex
TxIndex is used to look up transactions included in the blockchain by ID.
Definition: txindex.h:18
SER_DISK
@ SER_DISK
Definition: serialize.h:166
BaseIndex::DB
The database stores a block locator of the chain the database is synced to so that the TxIndex can ef...
Definition: base.h:36
CDBWrapper::Write
bool Write(const K &key, const V &value, bool fSync=false)
Definition: dbwrapper.h:256
error
bool error(const char *fmt, const Args &... args)
Definition: system.h:48
CBlockLocator
Describes a place in the block chain to another node such that if the other node doesn't have the sam...
Definition: block.h:100
CBlockIndex
The block chain is a tree shaped structure starting with the genesis block at the root,...
Definition: blockindex.h:23
CDBWrapper::CompactRange
void CompactRange(const K &key_begin, const K &key_end) const
Compact a certain range of keys in the database.
Definition: dbwrapper.h:318
CBlockHeader::GetHash
BlockHash GetHash() const
Definition: block.cpp:11
TxIndex::TxIndex
TxIndex(size_t n_cache_size, bool f_memory=false, bool f_wipe=false)
Constructs the index, which becomes available to be queried.
Definition: txindex.cpp:201
CDBWrapper::WriteBatch
bool WriteBatch(CDBBatch &batch, bool fSync=false)
Definition: dbwrapper.cpp:186
LogPrintf
static void LogPrintf(const char *fmt, const Args &... args)
Definition: logging.h:175
CDBBatch::SizeEstimate
size_t SizeEstimate() const
Definition: dbwrapper.h:112