47 void CheckUniqueFileid(
const BerkeleyEnvironment& env,
const std::string& filename, Db& db, WalletDatabaseFileId& fileid)
49 if (env.IsMock())
return;
51 int ret = db.get_mpf()->get_fileid(fileid.value);
53 throw std::runtime_error(
strprintf(
"BerkeleyDatabase: Can't open database %s (get_fileid failed with %d)", filename,
ret));
56 for (
const auto& item : env.m_fileids) {
57 if (fileid == item.second && &fileid != &item.second) {
58 throw std::runtime_error(
strprintf(
"BerkeleyDatabase: Can't open database %s (duplicates fileid %s from %s)", filename,
59 HexStr(item.second.value), item.first));
65 std::map<std::string, std::weak_ptr<BerkeleyEnvironment>> g_dbenvs
GUARDED_BY(cs_db);
82 auto inserted = g_dbenvs.emplace(
fs::PathToString(env_directory), std::weak_ptr<BerkeleyEnvironment>());
83 if (inserted.second) {
84 auto env = std::make_shared<BerkeleyEnvironment>(env_directory, use_shared_memory);
85 inserted.first->second = env;
88 return inserted.first->second.lock();
106 database.
m_db->close(0);
107 database.
m_db.reset();
111 FILE* error_file =
nullptr;
112 dbenv->get_errfile(&error_file);
116 LogPrintf(
"BerkeleyEnvironment::Close: Error %d closing database environment: %s\n",
ret, DbEnv::strerror(
ret));
118 DbEnv(uint32_t{0}).remove(
strPath.c_str(), 0);
120 if (error_file) fclose(error_file);
127 dbenv.reset(
new DbEnv(DB_CXX_NO_EXCEPTIONS));
153 LogPrintf(
"Cannot obtain a lock on wallet directory %s. Another instance may be using it.\n",
strPath);
158 fs::path pathLogDir = pathIn /
"database";
160 fs::path pathErrorFile = pathIn /
"db.log";
163 unsigned int nEnvFlags = 0;
165 nEnvFlags |= DB_PRIVATE;
169 dbenv->set_cachesize(0, 0x100000, 1);
170 dbenv->set_lg_bsize(0x10000);
171 dbenv->set_lg_max(1048576);
172 dbenv->set_lk_max_locks(40000);
173 dbenv->set_lk_max_objects(40000);
175 dbenv->set_flags(DB_AUTO_COMMIT, 1);
176 dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1);
177 dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1);
189 LogPrintf(
"BerkeleyEnvironment::Open: Error %d opening database environment: %s\n",
ret, DbEnv::strerror(
ret));
190 int ret2 =
dbenv->close(0);
192 LogPrintf(
"BerkeleyEnvironment::Open: Error %d closing failed database environment: %s\n", ret2, DbEnv::strerror(ret2));
196 if (
ret == DB_RUNRECOVERY) {
197 err +=
Untranslated(
" ") +
_(
"This error could occur if this wallet was not shutdown cleanly and was last loaded using a build with a newer version of Berkeley DB. If so, please use the software that last loaded this wallet");
214 dbenv->set_cachesize(1, 0, 1);
215 dbenv->set_lg_bsize(10485760 * 4);
216 dbenv->set_lg_max(10485760);
217 dbenv->set_lk_max_locks(10000);
218 dbenv->set_lk_max_objects(10000);
219 dbenv->set_flags(DB_AUTO_COMMIT, 1);
220 dbenv->log_set_config(DB_LOG_IN_MEMORY, 1);
231 throw std::runtime_error(
strprintf(
"BerkeleyEnvironment::MakeMock: Error %d opening database environment.",
ret));
247 SafeDbt(
void* data,
size_t size);
260 m_dbt.set_flags(DB_DBT_MALLOC);
270 if (
m_dbt.get_data() !=
nullptr) {
276 if (
m_dbt.get_flags() & DB_DBT_MALLOC) {
277 free(
m_dbt.get_data());
284 return m_dbt.get_data();
289 return m_dbt.get_size();
292 SafeDbt::operator Dbt*()
299 return {
reinterpret_cast<const std::byte*
>(dbt.
get_data()), dbt.
get_size()};
303 WalletDatabase(), env(
std::move(env)), m_filename(
std::move(filename)), m_max_log_mb(options.max_log_mb)
305 auto inserted = this->env->m_databases.emplace(
m_filename, std::ref(*
this));
317 if (!
env->Open(errorStr)) {
325 Db db(
env->dbenv.get(), 0);
327 int result = db.verify(strFile.c_str(),
nullptr,
nullptr, 0);
339 dbenv->txn_checkpoint(0, 0, 0);
342 dbenv->lsn_reset(strFile.c_str(), 0);
370 unsigned int nFlags = DB_THREAD | DB_CREATE;
375 if (!
env->Open(open_err))
376 throw std::runtime_error(
"BerkeleyDatabase: Failed to open database environment.");
378 if (
m_db ==
nullptr) {
380 std::unique_ptr<Db> pdb_temp = std::make_unique<Db>(
env->dbenv.get(), 0);
383 bool fMockDb =
env->IsMock();
385 DbMpoolFile* mpf = pdb_temp->get_mpf();
386 ret = mpf->set_flags(DB_MPOOL_NOFILE, 1);
388 throw std::runtime_error(
strprintf(
"BerkeleyDatabase: Failed to configure for no temp file backing for database %s", strFile));
392 ret = pdb_temp->open(
nullptr,
393 fMockDb ?
nullptr : strFile.c_str(),
394 fMockDb ? strFile.c_str() :
"main",
400 throw std::runtime_error(
strprintf(
"BerkeleyDatabase: Error %d, can't open database %s",
ret, strFile));
406 CheckUniqueFileid(*
env, strFile, *pdb_temp, this->env->m_fileids[strFile]);
408 m_db.reset(pdb_temp.release());
420 unsigned int nMinutes = 0;
462 database.
m_db->close(0);
463 database.
m_db.reset();
472 std::unique_lock<RecursiveMutex> lock(cs_db);
475 if (db.second.get().m_refcount > 0)
return false;
498 DbTxn* ptxn =
nullptr;
500 if (!ptxn ||
ret != 0)
514 env->CheckpointLSN(strFile);
517 bool fSuccess =
true;
518 LogPrintf(
"BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile);
519 std::string strFileRes = strFile +
".rewrite";
522 std::unique_ptr<Db> pdbCopy = std::make_unique<Db>(
env->dbenv.get(), 0);
524 int ret = pdbCopy->open(
nullptr,
531 LogPrintf(
"BerkeleyBatch::Rewrite: Can't create database file %s\n", strFileRes);
535 std::unique_ptr<DatabaseCursor> cursor = db.
GetNewCursor();
548 strncmp((
const char*)ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0)
550 if (strncmp((
const char*)ssKey.data(),
"\x07version", 8) == 0) {
555 Dbt datKey(ssKey.data(), ssKey.size());
556 Dbt datValue(ssValue.data(), ssValue.size());
557 int ret2 = pdbCopy->put(
nullptr, &datKey, &datValue, DB_NOOVERWRITE);
566 if (pdbCopy->close(0))
573 Db dbA(
env->dbenv.get(), 0);
574 if (dbA.remove(strFile.c_str(),
nullptr, 0))
576 Db dbB(
env->dbenv.get(), 0);
577 if (dbB.rename(strFileRes.c_str(),
nullptr, strFile.c_str(), 0))
581 LogPrintf(
"BerkeleyBatch::Rewrite: Failed to rewrite database file %s\n", strFileRes);
592 const auto start{SteadyClock::now()};
599 bool no_dbs_accessed =
true;
601 const fs::path& filename = db_it.first;
602 int nRefCount = db_it.second.get().m_refcount;
603 if (nRefCount < 0)
continue;
606 if (nRefCount == 0) {
610 dbenv->txn_checkpoint(0, 0, 0);
613 dbenv->lsn_reset(strFile.c_str(), 0);
617 no_dbs_accessed =
false;
620 LogPrint(
BCLog::WALLETDB,
"BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ?
"true" :
"false",
fDbEnvInit ?
"" :
" database not started", Ticks<std::chrono::milliseconds>(SteadyClock::now() - start));
623 if (no_dbs_accessed) {
624 dbenv->log_archive(&listp, DB_ARCH_REMOVE);
638 if (!lockDb)
return false;
641 for (
auto& it :
env->m_databases) {
642 if (it.second.get().m_refcount > 0)
return false;
650 const auto start{SteadyClock::now()};
654 env->CheckpointLSN(strFile);
657 LogPrint(
BCLog::WALLETDB,
"Flushed %s %dms\n", strFile, Ticks<std::chrono::milliseconds>(SteadyClock::now() - start));
673 env->CheckpointLSN(strFile);
678 if (fs::is_directory(pathDest))
682 if (
fs::exists(pathDest) && fs::equivalent(pathSrc, pathDest)) {
687 fs::copy_file(pathSrc, pathDest, fs::copy_options::overwrite_existing);
690 }
catch (
const fs::filesystem_error& e) {
718 if (!database.
m_db.get()) {
719 throw std::runtime_error(
STR_INTERNAL_BUG(
"BerkeleyDatabase does not exist"));
737 ret =
m_cursor->get(datKey, datValue, DB_SET_RANGE);
742 if (
ret == DB_NOTFOUND) {
756 ssKey.
write(raw_key);
771 if (!
pdb)
return nullptr;
772 return std::make_unique<BerkeleyCursor>(
m_database, *
this);
777 if (!
pdb)
return nullptr;
813 DbEnv::version(&major, &minor,
nullptr);
818 if (major != DB_VERSION_MAJOR || minor < DB_VERSION_MINOR) {
819 LogPrintf(
"BerkeleyDB database version conflict: header version is %d.%d, library version is %d.%d\n",
820 DB_VERSION_MAJOR, DB_VERSION_MINOR, major, minor);
829 return DbEnv::version(
nullptr,
nullptr,
nullptr);
837 SafeDbt datKey(key.data(), key.size());
854 assert(!
"Write called on database in read-only mode");
856 SafeDbt datKey(key.data(), key.size());
858 SafeDbt datValue(value.data(), value.size());
860 int ret =
pdb->put(
activeTxn, datKey, datValue, (overwrite ? 0 : DB_NOOVERWRITE));
869 assert(!
"Erase called on database in read-only mode");
871 SafeDbt datKey(key.data(), key.size());
874 return (
ret == 0 ||
ret == DB_NOTFOUND);
882 SafeDbt datKey(key.data(), key.size());
896 auto cursor{std::make_unique<BerkeleyCursor>(
m_database, *
this)};
900 Dbt prefix_key{
const_cast<std::byte*
>(
prefix.data()),
static_cast<uint32_t
>(
prefix.size())}, prefix_value{};
901 int ret{cursor->dbc()->get(&prefix_key, &prefix_value, DB_SET_RANGE)};
902 for (
int flag{DB_CURRENT};
ret == 0; flag = DB_NEXT) {
904 ret = cursor->dbc()->get(key, value, flag);
906 ret = cursor->dbc()->del(0);
909 return ret == 0 ||
ret == DB_NOTFOUND;
926 if (
env)
env->m_db_in_use.notify_all();
931 return std::make_unique<BerkeleyBatch>(*
this,
false, flush_on_close);
937 std::unique_ptr<BerkeleyDatabase> db;
942 if (env->m_databases.count(data_filename)) {
947 db = std::make_unique<BerkeleyDatabase>(std::move(env), std::move(data_filename), options);
950 if (options.
verify && !db->Verify(error)) {
#define BDB_DB_FILE_ID_LEN
#define STR_INTERNAL_BUG(msg)
#define Assume(val)
Assume is the identity function.
Double ended buffer combining vector and stream-like interfaces.
void write(Span< const value_type > src)
constexpr C * end() const noexcept
constexpr C * begin() const noexcept
Path class wrapper to block calls to the fs::path(std::string) implicit constructor and the fs::path:...
RAII class that provides access to a Berkeley database.
bool ReadKey(DataStream &&key, DataStream &value) override
bool HasKey(DataStream &&key) override
BerkeleyDatabase & m_database
bool EraseKey(DataStream &&key) override
~BerkeleyBatch() override
BerkeleyBatch(BerkeleyDatabase &database, const bool fReadOnly, bool fFlushOnCloseIn=true)
std::unique_ptr< DatabaseCursor > GetNewPrefixCursor(Span< const std::byte > prefix) override
bool TxnCommit() override
BerkeleyEnvironment * env
std::unique_ptr< DatabaseCursor > GetNewCursor() override
bool ErasePrefix(Span< const std::byte > prefix) override
bool WriteKey(DataStream &&key, DataStream &&value, bool overwrite=true) override
~BerkeleyCursor() override
std::vector< std::byte > m_key_prefix
BerkeleyCursor(BerkeleyDatabase &database, const BerkeleyBatch &batch, Span< const std::byte > prefix={})
Status Next(DataStream &key, DataStream &value) override
An instance of this class represents one database.
bool Rewrite(const char *pszSkip=nullptr) override
Rewrite the entire database on disk, with the exception of key pszSkip if non-zero.
void Open() override
Open the database if it is not already opened.
bool Backup(const std::string &strDest) const override
Back up the entire database to a file.
void RemoveRef() override
Indicate that database user has stopped using the database and that it could be flushed or closed.
std::shared_ptr< BerkeleyEnvironment > env
Pointer to shared database environment.
void IncrementUpdateCounter() override
void ReloadDbEnv() override
void AddRef() override
Indicate that a new database user has begun using the database.
BerkeleyDatabase()=delete
std::unique_ptr< Db > m_db
Database pointer.
void Flush() override
Make sure all changes are flushed to database file.
bool PeriodicFlush() override
std::unique_ptr< DatabaseBatch > MakeBatch(bool flush_on_close=true) override
Make a BerkeleyBatch connected to this database.
bool Verify(bilingual_str &error)
Verifies the environment and database file.
~BerkeleyDatabase() override
void Close() override
Flush to the database file and close the database.
fs::path Directory() const
std::condition_variable_any m_db_in_use
void CloseDb(const fs::path &filename)
std::unique_ptr< DbEnv > dbenv
BerkeleyEnvironment()
Construct an in-memory mock Berkeley environment for testing.
void CheckpointLSN(const std::string &strFile)
void Flush(bool fShutdown)
DbTxn * TxnBegin(int flags)
bool Open(bilingual_str &error)
std::map< fs::path, std::reference_wrapper< BerkeleyDatabase > > m_databases
RAII class that automatically cleanses its data on destruction.
const void * get_data() const
uint32_t get_size() const
An instance of this class represents one database.
std::atomic< unsigned int > nUpdateCounter
std::atomic< int > m_refcount
Counts the number of active database users to be sure that the database is not closed while someone i...
void memory_cleanse(void *ptr, size_t len)
Secure overwrite a buffer (possibly containing secret data) with zero-bytes.
static const int CLIENT_VERSION
bitcoind-res.rc includes this file, but it cannot cope with real c++ code.
bool TryCreateDirectories(const fs::path &p)
Ignores exceptions thrown by create_directories if the requested directory exists.
void UnlockDirectory(const fs::path &directory, const fs::path &lockfile_name)
#define LogPrint(category,...)
Filesystem operations and types.
static auto quoted(const std::string &s)
static bool exists(const path &p)
static bool copy_file(const path &from, const path &to, copy_options options)
static std::string PathToString(const path &path)
Convert path object to a byte string.
static path PathFromString(const std::string &string)
Convert byte string to path object.
FILE * fopen(const fs::path &p, const char *mode)
std::string get_filesystem_error_message(const fs::filesystem_error &e)
LockResult LockDirectory(const fs::path &directory, const fs::path &lockfile_name, bool probe_only)
std::unique_ptr< BerkeleyDatabase > MakeBerkeleyDatabase(const fs::path &path, const DatabaseOptions &options, DatabaseStatus &status, bilingual_str &error)
Return object giving access to Berkeley database at specified path.
static std::set< std::string > g_loading_wallet_set GUARDED_BY(g_loading_wallet_mutex)
std::string BerkeleyDatabaseVersion()
fs::path BDBDataFile(const fs::path &wallet_path)
std::shared_ptr< BerkeleyEnvironment > GetBerkeleyEnv(const fs::path &env_directory, bool use_shared_memory)
Get BerkeleyEnvironment given a directory path.
bool BerkeleyDatabaseSanityCheck()
Perform sanity check of runtime BDB version versus linked BDB version.
static Span< const std::byte > SpanFromDbt(const SafeDbt &dbt)
bool verify
Check data integrity on load.
bool use_shared_memory
Let other processes access the database.
bool operator==(const WalletDatabaseFileId &rhs) const
uint8_t value[BDB_DB_FILE_ID_LEN]
#define AssertLockNotHeld(cs)
#define TRY_LOCK(cs, name)
void UninterruptibleSleep(const std::chrono::microseconds &n)
bilingual_str _(const char *psz)
Translation function.
bilingual_str Untranslated(std::string original)
Mark a bilingual_str as untranslated.
static const char * filenames[]
std::string HexStr(const Span< const uint8_t > s)
Convert a span of bytes to a lower-case hexadecimal string.