Bitcoin Core  27.99.0
P2P Digital Currency
dbwrapper.cpp
Go to the documentation of this file.
1 // Copyright (c) 2012-2022 The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 
5 #include <dbwrapper.h>
6 
7 #include <logging.h>
8 #include <random.h>
9 #include <serialize.h>
10 #include <span.h>
11 #include <streams.h>
12 #include <util/fs.h>
13 #include <util/fs_helpers.h>
14 #include <util/strencodings.h>
15 
16 #include <algorithm>
17 #include <cassert>
18 #include <cstdarg>
19 #include <cstdint>
20 #include <cstdio>
21 #include <leveldb/cache.h>
22 #include <leveldb/db.h>
23 #include <leveldb/env.h>
24 #include <leveldb/filter_policy.h>
25 #include <leveldb/helpers/memenv/memenv.h>
26 #include <leveldb/iterator.h>
27 #include <leveldb/options.h>
28 #include <leveldb/slice.h>
29 #include <leveldb/status.h>
30 #include <leveldb/write_batch.h>
31 #include <memory>
32 #include <optional>
33 #include <utility>
34 
35 static auto CharCast(const std::byte* data) { return reinterpret_cast<const char*>(data); }
36 
37 bool DestroyDB(const std::string& path_str)
38 {
39  return leveldb::DestroyDB(path_str, {}).ok();
40 }
41 
44 static void HandleError(const leveldb::Status& status)
45 {
46  if (status.ok())
47  return;
48  const std::string errmsg = "Fatal LevelDB error: " + status.ToString();
49  LogPrintf("%s\n", errmsg);
50  LogPrintf("You can use -debug=leveldb to get more complete diagnostic messages\n");
51  throw dbwrapper_error(errmsg);
52 }
53 
54 class CBitcoinLevelDBLogger : public leveldb::Logger {
55 public:
56  // This code is adapted from posix_logger.h, which is why it is using vsprintf.
57  // Please do not do this in normal code
58  void Logv(const char * format, va_list ap) override {
60  return;
61  }
62  char buffer[500];
63  for (int iter = 0; iter < 2; iter++) {
64  char* base;
65  int bufsize;
66  if (iter == 0) {
67  bufsize = sizeof(buffer);
68  base = buffer;
69  }
70  else {
71  bufsize = 30000;
72  base = new char[bufsize];
73  }
74  char* p = base;
75  char* limit = base + bufsize;
76 
77  // Print the message
78  if (p < limit) {
79  va_list backup_ap;
80  va_copy(backup_ap, ap);
81  // Do not use vsnprintf elsewhere in bitcoin source code, see above.
82  p += vsnprintf(p, limit - p, format, backup_ap);
83  va_end(backup_ap);
84  }
85 
86  // Truncate to available space if necessary
87  if (p >= limit) {
88  if (iter == 0) {
89  continue; // Try again with larger buffer
90  }
91  else {
92  p = limit - 1;
93  }
94  }
95 
96  // Add newline if necessary
97  if (p == base || p[-1] != '\n') {
98  *p++ = '\n';
99  }
100 
101  assert(p <= limit);
102  base[std::min(bufsize - 1, (int)(p - base))] = '\0';
103  LogPrintLevel(BCLog::LEVELDB, BCLog::Level::Debug, "%s", base); // NOLINT(bitcoin-unterminated-logprintf)
104  if (base != buffer) {
105  delete[] base;
106  }
107  break;
108  }
109  }
110 };
111 
112 static void SetMaxOpenFiles(leveldb::Options *options) {
113  // On most platforms the default setting of max_open_files (which is 1000)
114  // is optimal. On Windows using a large file count is OK because the handles
115  // do not interfere with select() loops. On 64-bit Unix hosts this value is
116  // also OK, because up to that amount LevelDB will use an mmap
117  // implementation that does not use extra file descriptors (the fds are
118  // closed after being mmap'ed).
119  //
120  // Increasing the value beyond the default is dangerous because LevelDB will
121  // fall back to a non-mmap implementation when the file count is too large.
122  // On 32-bit Unix host we should decrease the value because the handles use
123  // up real fds, and we want to avoid fd exhaustion issues.
124  //
125  // See PR #12495 for further discussion.
126 
127  int default_open_files = options->max_open_files;
128 #ifndef WIN32
129  if (sizeof(void*) < 8) {
130  options->max_open_files = 64;
131  }
132 #endif
133  LogPrint(BCLog::LEVELDB, "LevelDB using max_open_files=%d (default=%d)\n",
134  options->max_open_files, default_open_files);
135 }
136 
137 static leveldb::Options GetOptions(size_t nCacheSize)
138 {
139  leveldb::Options options;
140  options.block_cache = leveldb::NewLRUCache(nCacheSize / 2);
141  options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously
142  options.filter_policy = leveldb::NewBloomFilterPolicy(10);
143  options.compression = leveldb::kNoCompression;
144  options.info_log = new CBitcoinLevelDBLogger();
145  if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
146  // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error
147  // on corruption in later versions.
148  options.paranoid_checks = true;
149  }
150  SetMaxOpenFiles(&options);
151  return options;
152 }
153 
155  leveldb::WriteBatch batch;
156 };
157 
159  : parent{_parent},
160  m_impl_batch{std::make_unique<CDBBatch::WriteBatchImpl>()} {};
161 
162 CDBBatch::~CDBBatch() = default;
163 
165 {
166  m_impl_batch->batch.Clear();
167  size_estimate = 0;
168 }
169 
171 {
172  leveldb::Slice slKey(CharCast(key.data()), key.size());
174  leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size());
175  m_impl_batch->batch.Put(slKey, slValue);
176  // LevelDB serializes writes as:
177  // - byte: header
178  // - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...)
179  // - byte[]: key
180  // - varint: value length
181  // - byte[]: value
182  // The formula below assumes the key and value are both less than 16k.
183  size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size();
184 }
185 
187 {
188  leveldb::Slice slKey(CharCast(key.data()), key.size());
189  m_impl_batch->batch.Delete(slKey);
190  // LevelDB serializes erases as:
191  // - byte: header
192  // - varint: key length
193  // - byte[]: key
194  // The formula below assumes the key is less than 16kB.
195  size_estimate += 2 + (slKey.size() > 127) + slKey.size();
196 }
197 
200  leveldb::Env* penv;
201 
203  leveldb::Options options;
204 
206  leveldb::ReadOptions readoptions;
207 
209  leveldb::ReadOptions iteroptions;
210 
212  leveldb::WriteOptions writeoptions;
213 
215  leveldb::WriteOptions syncoptions;
216 
218  leveldb::DB* pdb;
219 };
220 
222  : m_db_context{std::make_unique<LevelDBContext>()}, m_name{fs::PathToString(params.path.stem())}, m_path{params.path}, m_is_memory{params.memory_only}
223 {
224  DBContext().penv = nullptr;
225  DBContext().readoptions.verify_checksums = true;
226  DBContext().iteroptions.verify_checksums = true;
227  DBContext().iteroptions.fill_cache = false;
228  DBContext().syncoptions.sync = true;
229  DBContext().options = GetOptions(params.cache_bytes);
230  DBContext().options.create_if_missing = true;
231  if (params.memory_only) {
232  DBContext().penv = leveldb::NewMemEnv(leveldb::Env::Default());
233  DBContext().options.env = DBContext().penv;
234  } else {
235  if (params.wipe_data) {
236  LogPrintf("Wiping LevelDB in %s\n", fs::PathToString(params.path));
237  leveldb::Status result = leveldb::DestroyDB(fs::PathToString(params.path), DBContext().options);
238  HandleError(result);
239  }
240  TryCreateDirectories(params.path);
241  LogPrintf("Opening LevelDB in %s\n", fs::PathToString(params.path));
242  }
243  // PathToString() return value is safe to pass to leveldb open function,
244  // because on POSIX leveldb passes the byte string directly to ::open(), and
245  // on Windows it converts from UTF-8 to UTF-16 before calling ::CreateFileW
246  // (see env_posix.cc and env_windows.cc).
247  leveldb::Status status = leveldb::DB::Open(DBContext().options, fs::PathToString(params.path), &DBContext().pdb);
248  HandleError(status);
249  LogPrintf("Opened LevelDB successfully\n");
250 
251  if (params.options.force_compact) {
252  LogPrintf("Starting database compaction of %s\n", fs::PathToString(params.path));
253  DBContext().pdb->CompactRange(nullptr, nullptr);
254  LogPrintf("Finished database compaction of %s\n", fs::PathToString(params.path));
255  }
256 
257  // The base-case obfuscation key, which is a noop.
258  obfuscate_key = std::vector<unsigned char>(OBFUSCATE_KEY_NUM_BYTES, '\000');
259 
260  bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key);
261 
262  if (!key_exists && params.obfuscate && IsEmpty()) {
263  // Initialize non-degenerate obfuscation if it won't upset
264  // existing, non-obfuscated data.
265  std::vector<unsigned char> new_key = CreateObfuscateKey();
266 
267  // Write `new_key` so we don't obfuscate the key with itself
268  Write(OBFUSCATE_KEY_KEY, new_key);
269  obfuscate_key = new_key;
270 
271  LogPrintf("Wrote new obfuscate key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key));
272  }
273 
274  LogPrintf("Using obfuscation key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key));
275 }
276 
278 {
279  delete DBContext().pdb;
280  DBContext().pdb = nullptr;
281  delete DBContext().options.filter_policy;
282  DBContext().options.filter_policy = nullptr;
283  delete DBContext().options.info_log;
284  DBContext().options.info_log = nullptr;
285  delete DBContext().options.block_cache;
286  DBContext().options.block_cache = nullptr;
287  delete DBContext().penv;
288  DBContext().options.env = nullptr;
289 }
290 
291 bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync)
292 {
293  const bool log_memory = LogAcceptCategory(BCLog::LEVELDB, BCLog::Level::Debug);
294  double mem_before = 0;
295  if (log_memory) {
296  mem_before = DynamicMemoryUsage() / 1024.0 / 1024;
297  }
298  leveldb::Status status = DBContext().pdb->Write(fSync ? DBContext().syncoptions : DBContext().writeoptions, &batch.m_impl_batch->batch);
299  HandleError(status);
300  if (log_memory) {
301  double mem_after = DynamicMemoryUsage() / 1024.0 / 1024;
302  LogPrint(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n",
303  m_name, mem_before, mem_after);
304  }
305  return true;
306 }
307 
309 {
310  std::string memory;
311  std::optional<size_t> parsed;
312  if (!DBContext().pdb->GetProperty("leveldb.approximate-memory-usage", &memory) || !(parsed = ToIntegral<size_t>(memory))) {
313  LogPrint(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n");
314  return 0;
315  }
316  return parsed.value();
317 }
318 
319 // Prefixed with null character to avoid collisions with other keys
320 //
321 // We must use a string constructor which specifies length so that we copy
322 // past the null-terminator.
323 const std::string CDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14);
324 
325 const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8;
326 
331 std::vector<unsigned char> CDBWrapper::CreateObfuscateKey() const
332 {
333  std::vector<uint8_t> ret(OBFUSCATE_KEY_NUM_BYTES);
334  GetRandBytes(ret);
335  return ret;
336 }
337 
338 std::optional<std::string> CDBWrapper::ReadImpl(Span<const std::byte> key) const
339 {
340  leveldb::Slice slKey(CharCast(key.data()), key.size());
341  std::string strValue;
342  leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue);
343  if (!status.ok()) {
344  if (status.IsNotFound())
345  return std::nullopt;
346  LogPrintf("LevelDB read failure: %s\n", status.ToString());
347  HandleError(status);
348  }
349  return strValue;
350 }
351 
353 {
354  leveldb::Slice slKey(CharCast(key.data()), key.size());
355 
356  std::string strValue;
357  leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue);
358  if (!status.ok()) {
359  if (status.IsNotFound())
360  return false;
361  LogPrintf("LevelDB read failure: %s\n", status.ToString());
362  HandleError(status);
363  }
364  return true;
365 }
366 
368 {
369  leveldb::Slice slKey1(CharCast(key1.data()), key1.size());
370  leveldb::Slice slKey2(CharCast(key2.data()), key2.size());
371  uint64_t size = 0;
372  leveldb::Range range(slKey1, slKey2);
373  DBContext().pdb->GetApproximateSizes(&range, 1, &size);
374  return size;
375 }
376 
378 {
379  std::unique_ptr<CDBIterator> it(NewIterator());
380  it->SeekToFirst();
381  return !(it->Valid());
382 }
383 
385  const std::unique_ptr<leveldb::Iterator> iter;
386 
387  explicit IteratorImpl(leveldb::Iterator* _iter) : iter{_iter} {}
388 };
389 
390 CDBIterator::CDBIterator(const CDBWrapper& _parent, std::unique_ptr<IteratorImpl> _piter) : parent(_parent),
391  m_impl_iter(std::move(_piter)) {}
392 
394 {
395  return new CDBIterator{*this, std::make_unique<CDBIterator::IteratorImpl>(DBContext().pdb->NewIterator(DBContext().iteroptions))};
396 }
397 
399 {
400  leveldb::Slice slKey(CharCast(key.data()), key.size());
401  m_impl_iter->iter->Seek(slKey);
402 }
403 
405 {
406  return MakeByteSpan(m_impl_iter->iter->key());
407 }
408 
410 {
411  return MakeByteSpan(m_impl_iter->iter->value());
412 }
413 
414 CDBIterator::~CDBIterator() = default;
415 bool CDBIterator::Valid() const { return m_impl_iter->iter->Valid(); }
416 void CDBIterator::SeekToFirst() { m_impl_iter->iter->SeekToFirst(); }
417 void CDBIterator::Next() { m_impl_iter->iter->Next(); }
418 
419 namespace dbwrapper_private {
420 
421 const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w)
422 {
423  return w.obfuscate_key;
424 }
425 
426 } // namespace dbwrapper_private
int ret
void Logv(const char *format, va_list ap) override
Definition: dbwrapper.cpp:58
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:73
const std::unique_ptr< WriteBatchImpl > m_impl_batch
Definition: dbwrapper.h:79
size_t size_estimate
Definition: dbwrapper.h:85
DataStream ssValue
Definition: dbwrapper.h:83
void Clear()
Definition: dbwrapper.cpp:164
CDBBatch(const CDBWrapper &_parent)
Definition: dbwrapper.cpp:158
void WriteImpl(Span< const std::byte > key, DataStream &ssValue)
Definition: dbwrapper.cpp:170
const CDBWrapper & parent
Definition: dbwrapper.h:77
void EraseImpl(Span< const std::byte > key)
Definition: dbwrapper.cpp:186
Span< const std::byte > GetKeyImpl() const
Definition: dbwrapper.cpp:404
CDBIterator(const CDBWrapper &_parent, std::unique_ptr< IteratorImpl > _piter)
Definition: dbwrapper.cpp:390
const std::unique_ptr< IteratorImpl > m_impl_iter
Definition: dbwrapper.h:129
bool Valid() const
Definition: dbwrapper.cpp:415
void SeekToFirst()
Definition: dbwrapper.cpp:416
Span< const std::byte > GetValueImpl() const
Definition: dbwrapper.cpp:409
void Next()
Definition: dbwrapper.cpp:417
void SeekImpl(Span< const std::byte > key)
Definition: dbwrapper.cpp:398
auto & DBContext() const LIFETIMEBOUND
Definition: dbwrapper.h:211
size_t DynamicMemoryUsage() const
Definition: dbwrapper.cpp:308
bool WriteBatch(CDBBatch &batch, bool fSync=false)
Definition: dbwrapper.cpp:291
bool Read(const K &key, V &value) const
Definition: dbwrapper.h:221
CDBIterator * NewIterator()
Definition: dbwrapper.cpp:393
std::string m_name
the name of this database
Definition: dbwrapper.h:189
bool Write(const K &key, const V &value, bool fSync=false)
Definition: dbwrapper.h:241
std::vector< unsigned char > obfuscate_key
a key used for optional XOR-obfuscation of the database
Definition: dbwrapper.h:192
CDBWrapper(const DBParams &params)
Definition: dbwrapper.cpp:221
static const unsigned int OBFUSCATE_KEY_NUM_BYTES
the length of the obfuscate key in number of bytes
Definition: dbwrapper.h:198
static const std::string OBFUSCATE_KEY_KEY
the key under which the obfuscation key is stored
Definition: dbwrapper.h:195
std::optional< std::string > ReadImpl(Span< const std::byte > key) const
Definition: dbwrapper.cpp:338
std::vector< unsigned char > CreateObfuscateKey() const
Returns a string (consisting of 8 random bytes) suitable for use as an obfuscating XOR key.
Definition: dbwrapper.cpp:331
bool ExistsImpl(Span< const std::byte > key) const
Definition: dbwrapper.cpp:352
bool IsEmpty()
Return true if the database managed by this class contains no entries.
Definition: dbwrapper.cpp:377
size_t EstimateSizeImpl(Span< const std::byte > key1, Span< const std::byte > key2) const
Definition: dbwrapper.cpp:367
Double ended buffer combining vector and stream-like interfaces.
Definition: streams.h:147
size_type size() const
Definition: streams.h:181
value_type * data()
Definition: streams.h:188
void Xor(const std::vector< unsigned char > &key)
XOR the contents of this stream with a certain key.
Definition: streams.h:276
constexpr std::size_t size() const noexcept
Definition: span.h:187
constexpr C * data() const noexcept
Definition: span.h:174
static leveldb::Options GetOptions(size_t nCacheSize)
Definition: dbwrapper.cpp:137
static auto CharCast(const std::byte *data)
Definition: dbwrapper.cpp:35
bool DestroyDB(const std::string &path_str)
Definition: dbwrapper.cpp:37
static void SetMaxOpenFiles(leveldb::Options *options)
Definition: dbwrapper.cpp:112
static void HandleError(const leveldb::Status &status)
Handle database error by throwing dbwrapper_error exception.
Definition: dbwrapper.cpp:44
bool TryCreateDirectories(const fs::path &p)
Ignores exceptions thrown by create_directories if the requested directory exists.
Definition: fs_helpers.cpp:262
#define LogPrintLevel(category, level,...)
Definition: logging.h:252
#define LogPrint(category,...)
Definition: logging.h:264
static bool LogAcceptCategory(BCLog::LogFlags category, BCLog::Level level)
Return true if log accepts specified category, at the specified level.
Definition: logging.h:210
#define LogPrintf(...)
Definition: logging.h:245
@ LEVELDB
Definition: logging.h:61
These should be considered an implementation detail of the specific database.
Definition: dbwrapper.cpp:419
const std::vector< unsigned char > & GetObfuscateKey(const CDBWrapper &w)
Work around circular dependency, as well as for testing in dbwrapper_tests.
Definition: dbwrapper.cpp:421
Filesystem operations and types.
static std::string PathToString(const path &path)
Convert path object to a byte string.
Definition: fs.h:151
void format(std::ostream &out, const char *fmt, const Args &... args)
Format list of arguments to the stream according to given format string.
Definition: tinyformat.h:1060
void GetRandBytes(Span< unsigned char > bytes) noexcept
Overall design of the RNG and entropy sources.
Definition: random.cpp:638
Span< const std::byte > MakeByteSpan(V &&v) noexcept
Definition: span.h:277
leveldb::WriteBatch batch
Definition: dbwrapper.cpp:155
IteratorImpl(leveldb::Iterator *_iter)
Definition: dbwrapper.cpp:387
const std::unique_ptr< leveldb::Iterator > iter
Definition: dbwrapper.cpp:385
bool force_compact
Compact database on startup.
Definition: dbwrapper.h:29
Application-specific storage settings.
Definition: dbwrapper.h:33
DBOptions options
Passed-through options.
Definition: dbwrapper.h:46
bool obfuscate
If true, store data obfuscated via simple XOR.
Definition: dbwrapper.h:44
bool wipe_data
If true, remove all existing data.
Definition: dbwrapper.h:41
size_t cache_bytes
Configures various leveldb cache settings.
Definition: dbwrapper.h:37
fs::path path
Location in the filesystem where leveldb data will be stored.
Definition: dbwrapper.h:35
bool memory_only
If true, use leveldb's memory environment.
Definition: dbwrapper.h:39
leveldb::Env * penv
custom environment this database is using (may be nullptr in case of default environment)
Definition: dbwrapper.cpp:200
leveldb::ReadOptions iteroptions
options used when iterating over values of the database
Definition: dbwrapper.cpp:209
leveldb::ReadOptions readoptions
options used when reading from the database
Definition: dbwrapper.cpp:206
leveldb::Options options
database options used
Definition: dbwrapper.cpp:203
leveldb::DB * pdb
the database itself
Definition: dbwrapper.cpp:218
leveldb::WriteOptions syncoptions
options used when sync writing to the database
Definition: dbwrapper.cpp:215
leveldb::WriteOptions writeoptions
options used when writing to the database
Definition: dbwrapper.cpp:212
std::string HexStr(const Span< const uint8_t > s)
Convert a span of bytes to a lower-case hexadecimal string.
assert(!tx.IsCoinBase())