Bitcoin Core  22.99.0
P2P Digital Currency
dbwrapper.cpp
Go to the documentation of this file.
1 // Copyright (c) 2012-2021 The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 
5 #include <dbwrapper.h>
6 
7 #include <memory>
8 #include <random.h>
9 
10 #include <leveldb/cache.h>
11 #include <leveldb/env.h>
12 #include <leveldb/filter_policy.h>
13 #include <memenv.h>
14 #include <stdint.h>
15 #include <algorithm>
16 
17 class CBitcoinLevelDBLogger : public leveldb::Logger {
18 public:
19  // This code is adapted from posix_logger.h, which is why it is using vsprintf.
20  // Please do not do this in normal code
21  void Logv(const char * format, va_list ap) override {
23  return;
24  }
25  char buffer[500];
26  for (int iter = 0; iter < 2; iter++) {
27  char* base;
28  int bufsize;
29  if (iter == 0) {
30  bufsize = sizeof(buffer);
31  base = buffer;
32  }
33  else {
34  bufsize = 30000;
35  base = new char[bufsize];
36  }
37  char* p = base;
38  char* limit = base + bufsize;
39 
40  // Print the message
41  if (p < limit) {
42  va_list backup_ap;
43  va_copy(backup_ap, ap);
44  // Do not use vsnprintf elsewhere in bitcoin source code, see above.
45  p += vsnprintf(p, limit - p, format, backup_ap);
46  va_end(backup_ap);
47  }
48 
49  // Truncate to available space if necessary
50  if (p >= limit) {
51  if (iter == 0) {
52  continue; // Try again with larger buffer
53  }
54  else {
55  p = limit - 1;
56  }
57  }
58 
59  // Add newline if necessary
60  if (p == base || p[-1] != '\n') {
61  *p++ = '\n';
62  }
63 
64  assert(p <= limit);
65  base[std::min(bufsize - 1, (int)(p - base))] = '\0';
66  LogPrintf("leveldb: %s", base); /* Continued */
67  if (base != buffer) {
68  delete[] base;
69  }
70  break;
71  }
72  }
73 };
74 
75 static void SetMaxOpenFiles(leveldb::Options *options) {
76  // On most platforms the default setting of max_open_files (which is 1000)
77  // is optimal. On Windows using a large file count is OK because the handles
78  // do not interfere with select() loops. On 64-bit Unix hosts this value is
79  // also OK, because up to that amount LevelDB will use an mmap
80  // implementation that does not use extra file descriptors (the fds are
81  // closed after being mmap'ed).
82  //
83  // Increasing the value beyond the default is dangerous because LevelDB will
84  // fall back to a non-mmap implementation when the file count is too large.
85  // On 32-bit Unix host we should decrease the value because the handles use
86  // up real fds, and we want to avoid fd exhaustion issues.
87  //
88  // See PR #12495 for further discussion.
89 
90  int default_open_files = options->max_open_files;
91 #ifndef WIN32
92  if (sizeof(void*) < 8) {
93  options->max_open_files = 64;
94  }
95 #endif
96  LogPrint(BCLog::LEVELDB, "LevelDB using max_open_files=%d (default=%d)\n",
97  options->max_open_files, default_open_files);
98 }
99 
100 static leveldb::Options GetOptions(size_t nCacheSize)
101 {
102  leveldb::Options options;
103  options.block_cache = leveldb::NewLRUCache(nCacheSize / 2);
104  options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously
105  options.filter_policy = leveldb::NewBloomFilterPolicy(10);
106  options.compression = leveldb::kNoCompression;
107  options.info_log = new CBitcoinLevelDBLogger();
108  if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
109  // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error
110  // on corruption in later versions.
111  options.paranoid_checks = true;
112  }
113  SetMaxOpenFiles(&options);
114  return options;
115 }
116 
117 CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate)
118  : m_name{fs::PathToString(path.stem())}
119 {
120  penv = nullptr;
121  readoptions.verify_checksums = true;
122  iteroptions.verify_checksums = true;
123  iteroptions.fill_cache = false;
124  syncoptions.sync = true;
125  options = GetOptions(nCacheSize);
126  options.create_if_missing = true;
127  if (fMemory) {
128  penv = leveldb::NewMemEnv(leveldb::Env::Default());
129  options.env = penv;
130  } else {
131  if (fWipe) {
132  LogPrintf("Wiping LevelDB in %s\n", fs::PathToString(path));
133  leveldb::Status result = leveldb::DestroyDB(fs::PathToString(path), options);
135  }
136  TryCreateDirectories(path);
137  LogPrintf("Opening LevelDB in %s\n", fs::PathToString(path));
138  }
139  // PathToString() return value is safe to pass to leveldb open function,
140  // because on POSIX leveldb passes the byte string directly to ::open(), and
141  // on Windows it converts from UTF-8 to UTF-16 before calling ::CreateFileW
142  // (see env_posix.cc and env_windows.cc).
143  leveldb::Status status = leveldb::DB::Open(options, fs::PathToString(path), &pdb);
145  LogPrintf("Opened LevelDB successfully\n");
146 
147  if (gArgs.GetBoolArg("-forcecompactdb", false)) {
148  LogPrintf("Starting database compaction of %s\n", fs::PathToString(path));
149  pdb->CompactRange(nullptr, nullptr);
150  LogPrintf("Finished database compaction of %s\n", fs::PathToString(path));
151  }
152 
153  // The base-case obfuscation key, which is a noop.
154  obfuscate_key = std::vector<unsigned char>(OBFUSCATE_KEY_NUM_BYTES, '\000');
155 
156  bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key);
157 
158  if (!key_exists && obfuscate && IsEmpty()) {
159  // Initialize non-degenerate obfuscation if it won't upset
160  // existing, non-obfuscated data.
161  std::vector<unsigned char> new_key = CreateObfuscateKey();
162 
163  // Write `new_key` so we don't obfuscate the key with itself
164  Write(OBFUSCATE_KEY_KEY, new_key);
165  obfuscate_key = new_key;
166 
167  LogPrintf("Wrote new obfuscate key for %s: %s\n", fs::PathToString(path), HexStr(obfuscate_key));
168  }
169 
170  LogPrintf("Using obfuscation key for %s: %s\n", fs::PathToString(path), HexStr(obfuscate_key));
171 }
172 
174 {
175  delete pdb;
176  pdb = nullptr;
177  delete options.filter_policy;
178  options.filter_policy = nullptr;
179  delete options.info_log;
180  options.info_log = nullptr;
181  delete options.block_cache;
182  options.block_cache = nullptr;
183  delete penv;
184  options.env = nullptr;
185 }
186 
187 bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync)
188 {
189  const bool log_memory = LogAcceptCategory(BCLog::LEVELDB);
190  double mem_before = 0;
191  if (log_memory) {
192  mem_before = DynamicMemoryUsage() / 1024.0 / 1024;
193  }
194  leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch);
196  if (log_memory) {
197  double mem_after = DynamicMemoryUsage() / 1024.0 / 1024;
198  LogPrint(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n",
199  m_name, mem_before, mem_after);
200  }
201  return true;
202 }
203 
205 {
206  std::string memory;
207  std::optional<size_t> parsed;
208  if (!pdb->GetProperty("leveldb.approximate-memory-usage", &memory) || !(parsed = ToIntegral<size_t>(memory))) {
209  LogPrint(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n");
210  return 0;
211  }
212  return parsed.value();
213 }
214 
215 // Prefixed with null character to avoid collisions with other keys
216 //
217 // We must use a string constructor which specifies length so that we copy
218 // past the null-terminator.
219 const std::string CDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14);
220 
221 const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8;
222 
227 std::vector<unsigned char> CDBWrapper::CreateObfuscateKey() const
228 {
229  std::vector<uint8_t> ret(OBFUSCATE_KEY_NUM_BYTES);
231  return ret;
232 }
233 
235 {
236  std::unique_ptr<CDBIterator> it(NewIterator());
237  it->SeekToFirst();
238  return !(it->Valid());
239 }
240 
242 bool CDBIterator::Valid() const { return piter->Valid(); }
243 void CDBIterator::SeekToFirst() { piter->SeekToFirst(); }
244 void CDBIterator::Next() { piter->Next(); }
245 
246 namespace dbwrapper_private {
247 
248 void HandleError(const leveldb::Status& status)
249 {
250  if (status.ok())
251  return;
252  const std::string errmsg = "Fatal LevelDB error: " + status.ToString();
253  LogPrintf("%s\n", errmsg);
254  LogPrintf("You can use -debug=leveldb to get more complete diagnostic messages\n");
255  throw dbwrapper_error(errmsg);
256 }
257 
258 const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w)
259 {
260  return w.obfuscate_key;
261 }
262 
263 } // namespace dbwrapper_private
ArgsManager::GetBoolArg
bool GetBoolArg(const std::string &strArg, bool fDefault) const
Return boolean argument or default value.
Definition: system.cpp:597
dbwrapper.h
assert
assert(!tx.IsCoinBase())
tinyformat::format
void format(std::ostream &out, const char *fmt, const Args &... args)
Format list of arguments to the stream according to given format string.
Definition: tinyformat.h:1062
CDBBatch
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:47
fs::PathToString
static std::string PathToString(const path &path)
Convert path object to byte string.
Definition: fs.h:120
CBitcoinLevelDBLogger
Definition: dbwrapper.cpp:17
CDBWrapper::syncoptions
leveldb::WriteOptions syncoptions
options used when sync writing to the database
Definition: dbwrapper.h:196
SetMaxOpenFiles
static void SetMaxOpenFiles(leveldb::Options *options)
Definition: dbwrapper.cpp:75
CDBWrapper::CDBWrapper
CDBWrapper(const fs::path &path, size_t nCacheSize, bool fMemory=false, bool fWipe=false, bool obfuscate=false)
Definition: dbwrapper.cpp:117
GetOptions
static leveldb::Options GetOptions(size_t nCacheSize)
Definition: dbwrapper.cpp:100
CDBIterator::Next
void Next()
Definition: dbwrapper.cpp:244
random.h
CDBWrapper::NewIterator
CDBIterator * NewIterator()
Definition: dbwrapper.h:296
fs::path
Path class wrapper to prepare application code for transition from boost::filesystem library to std::...
Definition: fs.h:33
BCLog::LEVELDB
@ LEVELDB
Definition: logging.h:58
GetRandBytes
void GetRandBytes(unsigned char *buf, int num) noexcept
Overall design of the RNG and entropy sources.
Definition: random.cpp:584
CDBWrapper::penv
leveldb::Env * penv
custom environment this database is using (may be nullptr in case of default environment)
Definition: dbwrapper.h:181
CDBWrapper::OBFUSCATE_KEY_KEY
static const std::string OBFUSCATE_KEY_KEY
the key under which the obfuscation key is stored
Definition: dbwrapper.h:208
CDBIterator::~CDBIterator
~CDBIterator()
Definition: dbwrapper.cpp:241
LogPrintf
#define LogPrintf(...)
Definition: logging.h:187
CDBBatch::batch
leveldb::WriteBatch batch
Definition: dbwrapper.h:53
CDBWrapper::pdb
leveldb::DB * pdb
the database itself
Definition: dbwrapper.h:199
dbwrapper_private
These should be considered an implementation detail of the specific database.
Definition: dbwrapper.cpp:246
CDBIterator::piter
leveldb::Iterator * piter
Definition: dbwrapper.h:121
LogPrint
#define LogPrint(category,...)
Definition: logging.h:191
CDBWrapper::DynamicMemoryUsage
size_t DynamicMemoryUsage() const
Definition: dbwrapper.cpp:204
CDBWrapper::CreateObfuscateKey
std::vector< unsigned char > CreateObfuscateKey() const
Returns a string (consisting of 8 random bytes) suitable for use as an obfuscating XOR key.
Definition: dbwrapper.cpp:227
gArgs
ArgsManager gArgs
Definition: system.cpp:80
CDBWrapper::writeoptions
leveldb::WriteOptions writeoptions
options used when writing to the database
Definition: dbwrapper.h:193
CDBWrapper::m_name
std::string m_name
the name of this database
Definition: dbwrapper.h:202
dbwrapper_private::HandleError
void HandleError(const leveldb::Status &status)
Handle database error by throwing dbwrapper_error exception.
Definition: dbwrapper.cpp:248
CDBWrapper
Definition: dbwrapper.h:176
CDBIterator::Valid
bool Valid() const
Definition: dbwrapper.cpp:242
dbwrapper_error
Definition: dbwrapper.h:22
LogAcceptCategory
static bool LogAcceptCategory(BCLog::LogFlags category)
Return true if log accepts specified category.
Definition: logging.h:160
CDBWrapper::options
leveldb::Options options
database options used
Definition: dbwrapper.h:184
CDBWrapper::~CDBWrapper
~CDBWrapper()
Definition: dbwrapper.cpp:173
CDBWrapper::OBFUSCATE_KEY_NUM_BYTES
static const unsigned int OBFUSCATE_KEY_NUM_BYTES
the length of the obfuscate key in number of bytes
Definition: dbwrapper.h:211
CDBWrapper::obfuscate_key
std::vector< unsigned char > obfuscate_key
a key used for optional XOR-obfuscation of the database
Definition: dbwrapper.h:205
dbwrapper_private::GetObfuscateKey
const std::vector< unsigned char > & GetObfuscateKey(const CDBWrapper &w)
Work around circular dependency, as well as for testing in dbwrapper_tests.
Definition: dbwrapper.cpp:258
CDBIterator::SeekToFirst
void SeekToFirst()
Definition: dbwrapper.cpp:243
CDBWrapper::IsEmpty
bool IsEmpty()
Return true if the database managed by this class contains no entries.
Definition: dbwrapper.cpp:234
CBitcoinLevelDBLogger::Logv
void Logv(const char *format, va_list ap) override
Definition: dbwrapper.cpp:21
HexStr
std::string HexStr(const Span< const uint8_t > s)
Convert a span of bytes to a lower-case hexadecimal string.
Definition: strencodings.cpp:510
CDBWrapper::WriteBatch
bool WriteBatch(CDBBatch &batch, bool fSync=false)
Definition: dbwrapper.cpp:187
TryCreateDirectories
bool TryCreateDirectories(const fs::path &p)
Ignores exceptions thrown by Boost's create_directories if the requested directory exists.
Definition: system.cpp:1076