Bitcoin Core  24.99.0
P2P Digital Currency
dbwrapper.cpp
Go to the documentation of this file.
1 // Copyright (c) 2012-2022 The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 
5 #include <dbwrapper.h>
6 
7 #include <fs.h>
8 #include <logging.h>
9 #include <random.h>
10 #include <tinyformat.h>
11 #include <util/strencodings.h>
12 #include <util/system.h>
13 
14 #include <algorithm>
15 #include <cassert>
16 #include <cstdarg>
17 #include <cstdint>
18 #include <cstdio>
19 #include <leveldb/cache.h>
20 #include <leveldb/db.h>
21 #include <leveldb/env.h>
22 #include <leveldb/filter_policy.h>
23 #include <leveldb/helpers/memenv/memenv.h>
24 #include <leveldb/iterator.h>
25 #include <leveldb/options.h>
26 #include <leveldb/status.h>
27 #include <memory>
28 #include <optional>
29 
30 class CBitcoinLevelDBLogger : public leveldb::Logger {
31 public:
32  // This code is adapted from posix_logger.h, which is why it is using vsprintf.
33  // Please do not do this in normal code
34  void Logv(const char * format, va_list ap) override {
36  return;
37  }
38  char buffer[500];
39  for (int iter = 0; iter < 2; iter++) {
40  char* base;
41  int bufsize;
42  if (iter == 0) {
43  bufsize = sizeof(buffer);
44  base = buffer;
45  }
46  else {
47  bufsize = 30000;
48  base = new char[bufsize];
49  }
50  char* p = base;
51  char* limit = base + bufsize;
52 
53  // Print the message
54  if (p < limit) {
55  va_list backup_ap;
56  va_copy(backup_ap, ap);
57  // Do not use vsnprintf elsewhere in bitcoin source code, see above.
58  p += vsnprintf(p, limit - p, format, backup_ap);
59  va_end(backup_ap);
60  }
61 
62  // Truncate to available space if necessary
63  if (p >= limit) {
64  if (iter == 0) {
65  continue; // Try again with larger buffer
66  }
67  else {
68  p = limit - 1;
69  }
70  }
71 
72  // Add newline if necessary
73  if (p == base || p[-1] != '\n') {
74  *p++ = '\n';
75  }
76 
77  assert(p <= limit);
78  base[std::min(bufsize - 1, (int)(p - base))] = '\0';
79  LogPrintLevel(BCLog::LEVELDB, BCLog::Level::Debug, "%s", base); /* Continued */
80  if (base != buffer) {
81  delete[] base;
82  }
83  break;
84  }
85  }
86 };
87 
88 static void SetMaxOpenFiles(leveldb::Options *options) {
89  // On most platforms the default setting of max_open_files (which is 1000)
90  // is optimal. On Windows using a large file count is OK because the handles
91  // do not interfere with select() loops. On 64-bit Unix hosts this value is
92  // also OK, because up to that amount LevelDB will use an mmap
93  // implementation that does not use extra file descriptors (the fds are
94  // closed after being mmap'ed).
95  //
96  // Increasing the value beyond the default is dangerous because LevelDB will
97  // fall back to a non-mmap implementation when the file count is too large.
98  // On 32-bit Unix host we should decrease the value because the handles use
99  // up real fds, and we want to avoid fd exhaustion issues.
100  //
101  // See PR #12495 for further discussion.
102 
103  int default_open_files = options->max_open_files;
104 #ifndef WIN32
105  if (sizeof(void*) < 8) {
106  options->max_open_files = 64;
107  }
108 #endif
109  LogPrint(BCLog::LEVELDB, "LevelDB using max_open_files=%d (default=%d)\n",
110  options->max_open_files, default_open_files);
111 }
112 
113 static leveldb::Options GetOptions(size_t nCacheSize)
114 {
115  leveldb::Options options;
116  options.block_cache = leveldb::NewLRUCache(nCacheSize / 2);
117  options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously
118  options.filter_policy = leveldb::NewBloomFilterPolicy(10);
119  options.compression = leveldb::kNoCompression;
120  options.info_log = new CBitcoinLevelDBLogger();
121  if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
122  // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error
123  // on corruption in later versions.
124  options.paranoid_checks = true;
125  }
126  SetMaxOpenFiles(&options);
127  return options;
128 }
129 
130 CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate)
131  : m_name{fs::PathToString(path.stem())}, m_path{path}, m_is_memory{fMemory}
132 {
133  penv = nullptr;
134  readoptions.verify_checksums = true;
135  iteroptions.verify_checksums = true;
136  iteroptions.fill_cache = false;
137  syncoptions.sync = true;
138  options = GetOptions(nCacheSize);
139  options.create_if_missing = true;
140  if (fMemory) {
141  penv = leveldb::NewMemEnv(leveldb::Env::Default());
142  options.env = penv;
143  } else {
144  if (fWipe) {
145  LogPrintf("Wiping LevelDB in %s\n", fs::PathToString(path));
146  leveldb::Status result = leveldb::DestroyDB(fs::PathToString(path), options);
148  }
149  TryCreateDirectories(path);
150  LogPrintf("Opening LevelDB in %s\n", fs::PathToString(path));
151  }
152  // PathToString() return value is safe to pass to leveldb open function,
153  // because on POSIX leveldb passes the byte string directly to ::open(), and
154  // on Windows it converts from UTF-8 to UTF-16 before calling ::CreateFileW
155  // (see env_posix.cc and env_windows.cc).
156  leveldb::Status status = leveldb::DB::Open(options, fs::PathToString(path), &pdb);
158  LogPrintf("Opened LevelDB successfully\n");
159 
160  if (gArgs.GetBoolArg("-forcecompactdb", false)) {
161  LogPrintf("Starting database compaction of %s\n", fs::PathToString(path));
162  pdb->CompactRange(nullptr, nullptr);
163  LogPrintf("Finished database compaction of %s\n", fs::PathToString(path));
164  }
165 
166  // The base-case obfuscation key, which is a noop.
167  obfuscate_key = std::vector<unsigned char>(OBFUSCATE_KEY_NUM_BYTES, '\000');
168 
169  bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key);
170 
171  if (!key_exists && obfuscate && IsEmpty()) {
172  // Initialize non-degenerate obfuscation if it won't upset
173  // existing, non-obfuscated data.
174  std::vector<unsigned char> new_key = CreateObfuscateKey();
175 
176  // Write `new_key` so we don't obfuscate the key with itself
177  Write(OBFUSCATE_KEY_KEY, new_key);
178  obfuscate_key = new_key;
179 
180  LogPrintf("Wrote new obfuscate key for %s: %s\n", fs::PathToString(path), HexStr(obfuscate_key));
181  }
182 
183  LogPrintf("Using obfuscation key for %s: %s\n", fs::PathToString(path), HexStr(obfuscate_key));
184 }
185 
187 {
188  delete pdb;
189  pdb = nullptr;
190  delete options.filter_policy;
191  options.filter_policy = nullptr;
192  delete options.info_log;
193  options.info_log = nullptr;
194  delete options.block_cache;
195  options.block_cache = nullptr;
196  delete penv;
197  options.env = nullptr;
198 }
199 
200 bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync)
201 {
202  const bool log_memory = LogAcceptCategory(BCLog::LEVELDB, BCLog::Level::Debug);
203  double mem_before = 0;
204  if (log_memory) {
205  mem_before = DynamicMemoryUsage() / 1024.0 / 1024;
206  }
207  leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch);
209  if (log_memory) {
210  double mem_after = DynamicMemoryUsage() / 1024.0 / 1024;
211  LogPrint(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n",
212  m_name, mem_before, mem_after);
213  }
214  return true;
215 }
216 
218 {
219  std::string memory;
220  std::optional<size_t> parsed;
221  if (!pdb->GetProperty("leveldb.approximate-memory-usage", &memory) || !(parsed = ToIntegral<size_t>(memory))) {
222  LogPrint(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n");
223  return 0;
224  }
225  return parsed.value();
226 }
227 
228 // Prefixed with null character to avoid collisions with other keys
229 //
230 // We must use a string constructor which specifies length so that we copy
231 // past the null-terminator.
232 const std::string CDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14);
233 
234 const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8;
235 
240 std::vector<unsigned char> CDBWrapper::CreateObfuscateKey() const
241 {
242  std::vector<uint8_t> ret(OBFUSCATE_KEY_NUM_BYTES);
243  GetRandBytes(ret);
244  return ret;
245 }
246 
248 {
249  std::unique_ptr<CDBIterator> it(NewIterator());
250  it->SeekToFirst();
251  return !(it->Valid());
252 }
253 
255 bool CDBIterator::Valid() const { return piter->Valid(); }
256 void CDBIterator::SeekToFirst() { piter->SeekToFirst(); }
257 void CDBIterator::Next() { piter->Next(); }
258 
259 namespace dbwrapper_private {
260 
261 void HandleError(const leveldb::Status& status)
262 {
263  if (status.ok())
264  return;
265  const std::string errmsg = "Fatal LevelDB error: " + status.ToString();
266  LogPrintf("%s\n", errmsg);
267  LogPrintf("You can use -debug=leveldb to get more complete diagnostic messages\n");
268  throw dbwrapper_error(errmsg);
269 }
270 
271 const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w)
272 {
273  return w.obfuscate_key;
274 }
275 
276 } // namespace dbwrapper_private
int ret
bool GetBoolArg(const std::string &strArg, bool fDefault) const
Return boolean argument or default value.
Definition: system.cpp:641
void Logv(const char *format, va_list ap) override
Definition: dbwrapper.cpp:34
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:64
leveldb::WriteBatch batch
Definition: dbwrapper.h:69
leveldb::Iterator * piter
Definition: dbwrapper.h:137
bool Valid() const
Definition: dbwrapper.cpp:255
void SeekToFirst()
Definition: dbwrapper.cpp:256
void Next()
Definition: dbwrapper.cpp:257
CDBIterator * NewIterator()
Definition: dbwrapper.h:321
size_t DynamicMemoryUsage() const
Definition: dbwrapper.cpp:217
leveldb::Env * penv
custom environment this database is using (may be nullptr in case of default environment)
Definition: dbwrapper.h:192
bool WriteBatch(CDBBatch &batch, bool fSync=false)
Definition: dbwrapper.cpp:200
bool Read(const K &key, V &value) const
Definition: dbwrapper.h:248
std::string m_name
the name of this database
Definition: dbwrapper.h:213
bool Write(const K &key, const V &value, bool fSync=false)
Definition: dbwrapper.h:274
std::vector< unsigned char > obfuscate_key
a key used for optional XOR-obfuscation of the database
Definition: dbwrapper.h:216
leveldb::Options options
database options used
Definition: dbwrapper.h:195
static const unsigned int OBFUSCATE_KEY_NUM_BYTES
the length of the obfuscate key in number of bytes
Definition: dbwrapper.h:222
static const std::string OBFUSCATE_KEY_KEY
the key under which the obfuscation key is stored
Definition: dbwrapper.h:219
leveldb::WriteOptions writeoptions
options used when writing to the database
Definition: dbwrapper.h:204
leveldb::WriteOptions syncoptions
options used when sync writing to the database
Definition: dbwrapper.h:207
CDBWrapper(const fs::path &path, size_t nCacheSize, bool fMemory=false, bool fWipe=false, bool obfuscate=false)
Definition: dbwrapper.cpp:130
leveldb::DB * pdb
the database itself
Definition: dbwrapper.h:210
std::vector< unsigned char > CreateObfuscateKey() const
Returns a string (consisting of 8 random bytes) suitable for use as an obfuscating XOR key.
Definition: dbwrapper.cpp:240
leveldb::ReadOptions iteroptions
options used when iterating over values of the database
Definition: dbwrapper.h:201
bool IsEmpty()
Return true if the database managed by this class contains no entries.
Definition: dbwrapper.cpp:247
leveldb::ReadOptions readoptions
options used when reading from the database
Definition: dbwrapper.h:198
Path class wrapper to block calls to the fs::path(std::string) implicit constructor and the fs::path:...
Definition: fs.h:31
static leveldb::Options GetOptions(size_t nCacheSize)
Definition: dbwrapper.cpp:113
static void SetMaxOpenFiles(leveldb::Options *options)
Definition: dbwrapper.cpp:88
#define LogPrintLevel(category, level,...)
Definition: logging.h:253
#define LogPrint(category,...)
Definition: logging.h:245
static bool LogAcceptCategory(BCLog::LogFlags category, BCLog::Level level)
Return true if log accepts specified category, at the specified level.
Definition: logging.h:206
#define LogPrintf(...)
Definition: logging.h:236
@ LEVELDB
Definition: logging.h:60
These should be considered an implementation detail of the specific database.
Definition: dbwrapper.cpp:259
void HandleError(const leveldb::Status &status)
Handle database error by throwing dbwrapper_error exception.
Definition: dbwrapper.cpp:261
const std::vector< unsigned char > & GetObfuscateKey(const CDBWrapper &w)
Work around circular dependency, as well as for testing in dbwrapper_tests.
Definition: dbwrapper.cpp:271
Filesystem operations and types.
Definition: fs.h:20
static std::string PathToString(const path &path)
Convert path object to a byte string.
Definition: fs.h:150
void format(std::ostream &out, const char *fmt, const Args &... args)
Format list of arguments to the stream according to given format string.
Definition: tinyformat.h:1060
void GetRandBytes(Span< unsigned char > bytes) noexcept
Overall design of the RNG and entropy sources.
Definition: random.cpp:580
std::string HexStr(const Span< const uint8_t > s)
Convert a span of bytes to a lower-case hexadecimal string.
bool TryCreateDirectories(const fs::path &p)
Ignores exceptions thrown by create_directories if the requested directory exists.
Definition: system.cpp:1161
ArgsManager gArgs
Definition: system.cpp:73
assert(!tx.IsCoinBase())