txdb.cpp

Go to the documentation of this file.

// Copyright (c) 2009-2010 Satoshi Nakamoto
// Copyright (c) 2009-2018 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
#include <txdb.h>
 
#include <blockdb.h>
#include <chain.h>
#include <node/ui_interface.h>
#include <pow/pow.h>
#include <random.h>
#include <shutdown.h>
#include <util/system.h>
#include <util/translation.h>
#include <util/vector.h>
#include <version.h>
 
#include <cstdint>
#include <memory>
 
static const char DB_COIN = 'C';
static const char DB_COINS = 'c';
static const char DB_BLOCK_FILES = 'f';
static const char DB_BLOCK_INDEX = 'b';
 
static const char DB_BEST_BLOCK = 'B';
static const char DB_HEAD_BLOCKS = 'H';
static const char DB_FLAG = 'F';
static const char DB_REINDEX_FLAG = 'R';
static const char DB_LAST_BLOCK = 'l';
 
namespace {
 
struct CoinEntry {
    COutPoint *outpoint;
    char key;
    explicit CoinEntry(const COutPoint *ptr)
        : outpoint(const_cast<COutPoint *>(ptr)), key(DB_COIN) {}
 
    SERIALIZE_METHODS(CoinEntry, obj) {
        TxId id = obj.outpoint->GetTxId();
        uint32_t n = obj.outpoint->GetN();
        READWRITE(obj.key, id, VARINT(n));
        SER_READ(obj, *obj.outpoint = COutPoint(id, n));
    }
};
} // namespace
 
CCoinsViewDB::CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory,
                           bool fWipe)
    : m_db(std::make_unique<CDBWrapper>(ldb_path, nCacheSize, fMemory, fWipe,
                                        true)),
      m_ldb_path(ldb_path), m_is_memory(fMemory) {}
 
void CCoinsViewDB::ResizeCache(size_t new_cache_size) {
    // Have to do a reset first to get the original `m_db` state to release its
    // filesystem lock.
    m_db.reset();
    m_db = std::make_unique<CDBWrapper>(m_ldb_path, new_cache_size, m_is_memory,
                                        /*fWipe*/ false, /*obfuscate*/ true);
}
 
bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const {
    return m_db->Read(CoinEntry(&outpoint), coin);
}
 
bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const {
    return m_db->Exists(CoinEntry(&outpoint));
}
 
BlockHash CCoinsViewDB::GetBestBlock() const {
    BlockHash hashBestChain;
    if (!m_db->Read(DB_BEST_BLOCK, hashBestChain)) {
        return BlockHash();
    }
    return hashBestChain;
}
 
std::vector<BlockHash> CCoinsViewDB::GetHeadBlocks() const {
    std::vector<BlockHash> vhashHeadBlocks;
    if (!m_db->Read(DB_HEAD_BLOCKS, vhashHeadBlocks)) {
        return std::vector<BlockHash>();
    }
    return vhashHeadBlocks;
}
 
bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const BlockHash &hashBlock) {
    CDBBatch batch(*m_db);
    size_t count = 0;
    size_t changed = 0;
    size_t batch_size =
        (size_t)gArgs.GetArg("-dbbatchsize", DEFAULT_DB_BATCH_SIZE);
    int crash_simulate = gArgs.GetArg("-dbcrashratio", 0);
    assert(!hashBlock.IsNull());
 
    BlockHash old_tip = GetBestBlock();
    if (old_tip.IsNull()) {
        // We may be in the middle of replaying.
        std::vector<BlockHash> old_heads = GetHeadBlocks();
        if (old_heads.size() == 2) {
            assert(old_heads[0] == hashBlock);
            old_tip = old_heads[1];
        }
    }
 
    // In the first batch, mark the database as being in the middle of a
    // transition from old_tip to hashBlock.
    // A vector is used for future extensibility, as we may want to support
    // interrupting after partial writes from multiple independent reorgs.
    batch.Erase(DB_BEST_BLOCK);
    batch.Write(DB_HEAD_BLOCKS, Vector(hashBlock, old_tip));
 
    for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) {
        if (it->second.flags & CCoinsCacheEntry::DIRTY) {
            CoinEntry entry(&it->first);
            if (it->second.coin.IsSpent()) {
                batch.Erase(entry);
            } else {
                batch.Write(entry, it->second.coin);
            }
            changed++;
        }
        count++;
        CCoinsMap::iterator itOld = it++;
        mapCoins.erase(itOld);
        if (batch.SizeEstimate() > batch_size) {
            LogPrint(BCLog::COINDB, "Writing partial batch of %.2f MiB\n",
                     batch.SizeEstimate() * (1.0 / 1048576.0));
            m_db->WriteBatch(batch);
            batch.Clear();
            if (crash_simulate) {
                static FastRandomContext rng;
                if (rng.randrange(crash_simulate) == 0) {
                    LogPrintf("Simulating a crash. Goodbye.\n");
                    _Exit(0);
                }
            }
        }
    }
 
    // In the last batch, mark the database as consistent with hashBlock again.
    batch.Erase(DB_HEAD_BLOCKS);
    batch.Write(DB_BEST_BLOCK, hashBlock);
 
    LogPrint(BCLog::COINDB, "Writing final batch of %.2f MiB\n",
             batch.SizeEstimate() * (1.0 / 1048576.0));
    bool ret = m_db->WriteBatch(batch);
    LogPrint(BCLog::COINDB,
             "Committed %u changed transaction outputs (out of "
             "%u) to coin database...\n",
             (unsigned int)changed, (unsigned int)count);
    return ret;
}
 
size_t CCoinsViewDB::EstimateSize() const {
    return m_db->EstimateSize(DB_COIN, char(DB_COIN + 1));
}
 
CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe)
    : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory,
                 fWipe) {}
 
bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) {
    return Read(std::make_pair(DB_BLOCK_FILES, nFile), info);
}
 
bool CBlockTreeDB::WriteReindexing(bool fReindexing) {
    if (fReindexing) {
        return Write(DB_REINDEX_FLAG, '1');
    } else {
        return Erase(DB_REINDEX_FLAG);
    }
}
 
bool CBlockTreeDB::IsReindexing() const {
    return Exists(DB_REINDEX_FLAG);
}
 
bool CBlockTreeDB::ReadLastBlockFile(int &nFile) {
    return Read(DB_LAST_BLOCK, nFile);
}
 
CCoinsViewCursor *CCoinsViewDB::Cursor() const {
    CCoinsViewDBCursor *i = new CCoinsViewDBCursor(
        const_cast<CDBWrapper &>(*m_db).NewIterator(), GetBestBlock());
    i->pcursor->Seek(DB_COIN);
    // Cache key of first record
    if (i->pcursor->Valid()) {
        CoinEntry entry(&i->keyTmp.second);
        i->pcursor->GetKey(entry);
        i->keyTmp.first = entry.key;
    } else {
        // Make sure Valid() and GetKey() return false
        i->keyTmp.first = 0;
    }
    return i;
}
 
bool CCoinsViewDBCursor::GetKey(COutPoint &key) const {
    // Return cached key
    if (keyTmp.first == DB_COIN) {
        key = keyTmp.second;
        return true;
    }
    return false;
}
 
bool CCoinsViewDBCursor::GetValue(Coin &coin) const {
    return pcursor->GetValue(coin);
}
 
unsigned int CCoinsViewDBCursor::GetValueSize() const {
    return pcursor->GetValueSize();
}
 
bool CCoinsViewDBCursor::Valid() const {
    return keyTmp.first == DB_COIN;
}
 
void CCoinsViewDBCursor::Next() {
    pcursor->Next();
    CoinEntry entry(&keyTmp.second);
    if (!pcursor->Valid() || !pcursor->GetKey(entry)) {
        // Invalidate cached key after last record so that Valid() and GetKey()
        // return false
        keyTmp.first = 0;
    } else {
        keyTmp.first = entry.key;
    }
}
 
bool CBlockTreeDB::WriteBatchSync(
    const std::vector<std::pair<int, const CBlockFileInfo *>> &fileInfo,
    int nLastFile, const std::vector<const CBlockIndex *> &blockinfo) {
    CDBBatch batch(*this);
    for (std::vector<std::pair<int, const CBlockFileInfo *>>::const_iterator
             it = fileInfo.begin();
         it != fileInfo.end(); it++) {
        batch.Write(std::make_pair(DB_BLOCK_FILES, it->first), *it->second);
    }
    batch.Write(DB_LAST_BLOCK, nLastFile);
    for (std::vector<const CBlockIndex *>::const_iterator it =
             blockinfo.begin();
         it != blockinfo.end(); it++) {
        batch.Write(std::make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash()),
                    CDiskBlockIndex(*it));
    }
    return WriteBatch(batch, true);
}
 
bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) {
    return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0');
}
 
bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) {
    char ch;
    if (!Read(std::make_pair(DB_FLAG, name), ch)) {
        return false;
    }
    fValue = ch == '1';
    return true;
}
 
bool CBlockTreeDB::LoadBlockIndexGuts(
    const Consensus::Params &params,
    std::function<CBlockIndex *(const BlockHash &)> insertBlockIndex) {
    std::unique_ptr<CDBIterator> pcursor(NewIterator());
 
    uint64_t version = 0;
    pcursor->Seek("version");
    if (pcursor->Valid()) {
        pcursor->GetValue(version);
    }
 
    if (version != CLIENT_VERSION) {
        return error("%s: Invalid block index database version: %s", __func__,
                     version);
    }
 
    pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256()));
 
    // Load m_block_index
    while (pcursor->Valid()) {
        if (ShutdownRequested()) {
            return false;
        }
        std::pair<char, uint256> key;
        if (!pcursor->GetKey(key) || key.first != DB_BLOCK_INDEX) {
            break;
        }
 
        CDiskBlockIndex diskindex;
        if (!pcursor->GetValue(diskindex)) {
            return error("%s : failed to read value", __func__);
        }
 
        // Construct block index object
        CBlockIndex *pindexNew = insertBlockIndex(diskindex.GetBlockHash());
        pindexNew->pprev = insertBlockIndex(diskindex.hashPrev);
        pindexNew->nHeight = diskindex.nHeight;
        pindexNew->nFile = diskindex.nFile;
        pindexNew->nDataPos = diskindex.nDataPos;
        pindexNew->nUndoPos = diskindex.nUndoPos;
        pindexNew->nVersion = diskindex.nVersion;
        pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot;
        pindexNew->nTime = diskindex.nTime;
        pindexNew->nBits = diskindex.nBits;
        pindexNew->nNonce = diskindex.nNonce;
        pindexNew->nStatus = diskindex.nStatus;
        pindexNew->nTx = diskindex.nTx;
 
        if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits,
                              params)) {
            return error("%s: CheckProofOfWork failed: %s", __func__,
                         pindexNew->ToString());
        }
 
        pcursor->Next();
    }
 
    return true;
}
 
namespace {
class CCoins {
public:
    bool fCoinBase;
 
    std::vector<CTxOut> vout;
 
    int nHeight;
 
    CCoins() : fCoinBase(false), vout(0), nHeight(0) {}
 
    template <typename Stream> void Unserialize(Stream &s) {
        uint32_t nCode = 0;
        // version
        unsigned int nVersionDummy = 0;
        ::Unserialize(s, VARINT(nVersionDummy));
        // header code
        ::Unserialize(s, VARINT(nCode));
        fCoinBase = nCode & 1;
        std::vector<bool> vAvail(2, false);
        vAvail[0] = (nCode & 2) != 0;
        vAvail[1] = (nCode & 4) != 0;
        uint32_t nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1);
        // spentness bitmask
        while (nMaskCode > 0) {
            uint8_t chAvail = 0;
            ::Unserialize(s, chAvail);
            for (unsigned int p = 0; p < 8; p++) {
                bool f = (chAvail & (1 << p)) != 0;
                vAvail.push_back(f);
            }
            if (chAvail != 0) {
                nMaskCode--;
            }
        }
        // txouts themself
        vout.assign(vAvail.size(), CTxOut());
        for (size_t i = 0; i < vAvail.size(); i++) {
            if (vAvail[i]) {
                ::Unserialize(s, Using<TxOutCompression>(vout[i]));
            }
        }
        // coinbase height
        ::Unserialize(s, VARINT_MODE(nHeight, VarIntMode::NONNEGATIVE_SIGNED));
    }
};
} // namespace
 
bool CCoinsViewDB::Upgrade() {
    std::unique_ptr<CDBIterator> pcursor(m_db->NewIterator());
    pcursor->Seek(std::make_pair(DB_COINS, uint256()));
    if (!pcursor->Valid()) {
        return true;
    }
 
    int64_t count = 0;
    LogPrintf("Upgrading utxo-set database...\n");
    size_t batch_size = 1 << 24;
    CDBBatch batch(*m_db);
    int reportDone = -1;
    std::pair<uint8_t, uint256> key;
    std::pair<uint8_t, uint256> prev_key = {DB_COINS, uint256()};
    while (pcursor->Valid()) {
        if (ShutdownRequested()) {
            break;
        }
 
        if (!pcursor->GetKey(key) || key.first != DB_COINS) {
            break;
        }
 
        if (count++ % 256 == 0) {
            uint32_t high =
                0x100 * *key.second.begin() + *(key.second.begin() + 1);
            int percentageDone = (int)(high * 100.0 / 65536.0 + 0.5);
            uiInterface.ShowProgress(_("Upgrading UTXO database").translated,
                                     percentageDone, true);
            if (reportDone < percentageDone / 10) {
                // report max. every 10% step
                LogPrintfToBeContinued("[%d%%]...", percentageDone);
                reportDone = percentageDone / 10;
            }
        }
 
        CCoins old_coins;
        if (!pcursor->GetValue(old_coins)) {
            return error("%s: cannot parse CCoins record", __func__);
        }
 
        const TxId id(key.second);
        for (size_t i = 0; i < old_coins.vout.size(); ++i) {
            if (!old_coins.vout[i].IsNull() &&
                !old_coins.vout[i].scriptPubKey.IsUnspendable()) {
                Coin newcoin(std::move(old_coins.vout[i]), old_coins.nHeight,
                             old_coins.fCoinBase);
                COutPoint outpoint(id, i);
                CoinEntry entry(&outpoint);
                batch.Write(entry, newcoin);
            }
        }
 
        batch.Erase(key);
        if (batch.SizeEstimate() > batch_size) {
            m_db->WriteBatch(batch);
            batch.Clear();
            m_db->CompactRange(prev_key, key);
            prev_key = key;
        }
 
        pcursor->Next();
    }
 
    m_db->WriteBatch(batch);
    m_db->CompactRange({DB_COINS, uint256()}, key);
    uiInterface.ShowProgress("", 100, false);
    LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE");
    return !ShutdownRequested();
}
 
bool CBlockTreeDB::Upgrade(const Consensus::Params &params) {
    std::unique_ptr<CDBIterator> pcursor(NewIterator());
 
    uint64_t version = 0;
    pcursor->Seek("version");
    if (pcursor->Valid()) {
        pcursor->GetValue(version);
    }
 
    if (version >= CLIENT_VERSION) {
        // The DB is already up to date.
        return true;
    }
 
    CDBBatch batch(*this);
 
    pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256()));
    if (!pcursor->Valid()) {
        // The DB is empty, so just write the version number and consider the
        // upgrade done.
        batch.Write("version", uint64_t(CLIENT_VERSION));
        WriteBatch(batch);
        return true;
    }
 
    int64_t count = 0;
    LogPrintf("Upgrading block index database...\n");
    int reportDone = -1;
    std::pair<uint8_t, uint256> key = {DB_BLOCK_INDEX, uint256()};
    while (pcursor->Valid()) {
        if (ShutdownRequested()) {
            break;
        }
 
        if (!pcursor->GetKey(key) || key.first != DB_BLOCK_INDEX) {
            break;
        }
 
        if (count++ % 256 == 0) {
            uint32_t high =
                0x100 * *key.second.begin() + *(key.second.begin() + 1);
            int percentageDone = (int)(high * 100.0 / 65536.0 + 0.5);
            uiInterface.ShowProgress(
                _("Upgrading block index database").translated, percentageDone,
                true);
            if (reportDone < percentageDone / 10) {
                // report max. every 10% step
                LogPrintfToBeContinued("[%d%%]...", percentageDone);
                reportDone = percentageDone / 10;
            }
        }
 
        // Read the block index entry and update it.
        CDiskBlockIndex diskindex;
        if (!pcursor->GetValue(diskindex)) {
            return error("%s: cannot parse CDiskBlockIndex record", __func__);
        }
 
        // The block hash needs to be usable.
        BlockHash blockhash = diskindex.GetBlockHash();
        diskindex.phashBlock = &blockhash;
 
        bool mustUpdate = false;
 
        // We must update the block index to add the size.
        if (CLIENT_VERSION >= CDiskBlockIndex::TRACK_SIZE_VERSION &&
            version < CDiskBlockIndex::TRACK_SIZE_VERSION &&
            diskindex.nTx > 0 && diskindex.nSize == 0) {
            if (!diskindex.nStatus.hasData()) {
                // The block was pruned, we need a full reindex.
                LogPrintf("\nThe block %s is pruned. The block index cannot be "
                          "upgraded and reindexing is required.\n",
                          blockhash.GetHex());
                return false;
            }
 
            CBlock block;
            if (!ReadBlockFromDisk(block, &diskindex, params)) {
                // Failed to read the block from disk, even though it is marked
                // that we have data for this block.
                return false;
            }
 
            mustUpdate = true;
            diskindex.nSize = ::GetSerializeSize(block, PROTOCOL_VERSION);
        }
 
        if (mustUpdate) {
            batch.Write(std::make_pair(DB_BLOCK_INDEX, blockhash), diskindex);
        }
 
        pcursor->Next();
    }
 
    // Upgrade is done, now let's update the version number.
    batch.Write("version", uint64_t(CLIENT_VERSION));
 
    WriteBatch(batch);
    CompactRange({DB_BLOCK_INDEX, uint256()}, key);
    uiInterface.ShowProgress("", 100, false);
    LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE");
    return !ShutdownRequested();
}