Bitcoin ABC  0.24.10
P2P Digital Currency
dbwrapper.cpp
Go to the documentation of this file.
1 // Copyright (c) 2012-2016 The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 
5 #include <dbwrapper.h>
6 
7 #include <random.h>
8 
9 #include <leveldb/cache.h>
10 #include <leveldb/env.h>
11 #include <leveldb/filter_policy.h>
12 #include <memenv.h>
13 
14 #include <algorithm>
15 #include <cstdint>
16 #include <memory>
17 
18 class CBitcoinLevelDBLogger : public leveldb::Logger {
19 public:
20  // This code is adapted from posix_logger.h, which is why it is using
21  // vsprintf.
22  // Please do not do this in normal code
23  void Logv(const char *format, va_list ap) override {
25  return;
26  }
27  char buffer[500];
28  for (int iter = 0; iter < 2; iter++) {
29  char *base;
30  int bufsize;
31  if (iter == 0) {
32  bufsize = sizeof(buffer);
33  base = buffer;
34  } else {
35  bufsize = 30000;
36  base = new char[bufsize];
37  }
38  char *p = base;
39  char *limit = base + bufsize;
40 
41  // Print the message
42  if (p < limit) {
43  va_list backup_ap;
44  va_copy(backup_ap, ap);
45  // Do not use vsnprintf elsewhere in bitcoin source code, see
46  // above.
47  p += vsnprintf(p, limit - p, format, backup_ap);
48  va_end(backup_ap);
49  }
50 
51  // Truncate to available space if necessary
52  if (p >= limit) {
53  if (iter == 0) {
54  continue; // Try again with larger buffer
55  } else {
56  p = limit - 1;
57  }
58  }
59 
60  // Add newline if necessary
61  if (p == base || p[-1] != '\n') {
62  *p++ = '\n';
63  }
64 
65  assert(p <= limit);
66  base[std::min(bufsize - 1, (int)(p - base))] = '\0';
67  LogPrintfToBeContinued("leveldb: %s", base);
68  if (base != buffer) {
69  delete[] base;
70  }
71  break;
72  }
73  }
74 };
75 
76 static void SetMaxOpenFiles(leveldb::Options *options) {
77  // On most platforms the default setting of max_open_files (which is 1000)
78  // is optimal. On Windows using a large file count is OK because the handles
79  // do not interfere with select() loops. On 64-bit Unix hosts this value is
80  // also OK, because up to that amount LevelDB will use an mmap
81  // implementation that does not use extra file descriptors (the fds are
82  // closed after being mmaped).
83  //
84  // Increasing the value beyond the default is dangerous because LevelDB will
85  // fall back to a non-mmap implementation when the file count is too large.
86  // On 32-bit Unix host we should decrease the value because the handles use
87  // up real fds, and we want to avoid fd exhaustion issues.
88  //
89  // See PR #12495 for further discussion.
90 
91  int default_open_files = options->max_open_files;
92 #ifndef WIN32
93  if (sizeof(void *) < 8) {
94  options->max_open_files = 64;
95  }
96 #endif
97  LogPrint(BCLog::LEVELDB, "LevelDB using max_open_files=%d (default=%d)\n",
98  options->max_open_files, default_open_files);
99 }
100 
101 static leveldb::Options GetOptions(size_t nCacheSize) {
102  leveldb::Options options;
103  options.block_cache = leveldb::NewLRUCache(nCacheSize / 2);
104  // up to two write buffers may be held in memory simultaneously
105  options.write_buffer_size = nCacheSize / 4;
106  options.filter_policy = leveldb::NewBloomFilterPolicy(10);
107  options.compression = leveldb::kNoCompression;
108  options.info_log = new CBitcoinLevelDBLogger();
109  if (leveldb::kMajorVersion > 1 ||
110  (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
111  // LevelDB versions before 1.16 consider short writes to be corruption.
112  // Only trigger error on corruption in later versions.
113  options.paranoid_checks = true;
114  }
115  SetMaxOpenFiles(&options);
116  return options;
117 }
118 
119 CDBWrapper::CDBWrapper(const fs::path &path, size_t nCacheSize, bool fMemory,
120  bool fWipe, bool obfuscate)
121  : m_name{fs::PathToString(path.stem())} {
122  penv = nullptr;
123  readoptions.verify_checksums = true;
124  iteroptions.verify_checksums = true;
125  iteroptions.fill_cache = false;
126  syncoptions.sync = true;
127  options = GetOptions(nCacheSize);
128  options.create_if_missing = true;
129  if (fMemory) {
130  penv = leveldb::NewMemEnv(leveldb::Env::Default());
131  options.env = penv;
132  } else {
133  if (fWipe) {
134  LogPrintf("Wiping LevelDB in %s\n", fs::PathToString(path));
135  leveldb::Status result =
136  leveldb::DestroyDB(fs::PathToString(path), options);
138  }
139  TryCreateDirectories(path);
140  LogPrintf("Opening LevelDB in %s\n", fs::PathToString(path));
141  }
142  leveldb::Status status =
143  leveldb::DB::Open(options, fs::PathToString(path), &pdb);
145  LogPrintf("Opened LevelDB successfully\n");
146 
147  if (gArgs.GetBoolArg("-forcecompactdb", false)) {
148  LogPrintf("Starting database compaction of %s\n",
149  fs::PathToString(path));
150  pdb->CompactRange(nullptr, nullptr);
151  LogPrintf("Finished database compaction of %s\n",
152  fs::PathToString(path));
153  }
154 
155  // The base-case obfuscation key, which is a noop.
156  obfuscate_key = std::vector<uint8_t>(OBFUSCATE_KEY_NUM_BYTES, '\000');
157 
158  bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key);
159 
160  if (!key_exists && obfuscate && IsEmpty()) {
161  // Initialize non-degenerate obfuscation if it won't upset existing,
162  // non-obfuscated data.
163  std::vector<uint8_t> new_key = CreateObfuscateKey();
164 
165  // Write `new_key` so we don't obfuscate the key with itself
166  Write(OBFUSCATE_KEY_KEY, new_key);
167  obfuscate_key = new_key;
168 
169  LogPrintf("Wrote new obfuscate key for %s: %s\n",
170  fs::PathToString(path), HexStr(obfuscate_key));
171  }
172 
173  LogPrintf("Using obfuscation key for %s: %s\n", fs::PathToString(path),
174  HexStr(obfuscate_key));
175 }
176 
178  delete pdb;
179  pdb = nullptr;
180  delete options.filter_policy;
181  options.filter_policy = nullptr;
182  delete options.info_log;
183  options.info_log = nullptr;
184  delete options.block_cache;
185  options.block_cache = nullptr;
186  delete penv;
187  options.env = nullptr;
188 }
189 
190 bool CDBWrapper::WriteBatch(CDBBatch &batch, bool fSync) {
191  const bool log_memory = LogAcceptCategory(BCLog::LEVELDB);
192  double mem_before = 0;
193  if (log_memory) {
194  mem_before = DynamicMemoryUsage() / 1024.0 / 1024;
195  }
196  leveldb::Status status =
197  pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch);
199  if (log_memory) {
200  double mem_after = DynamicMemoryUsage() / 1024.0 / 1024;
201  LogPrint(
203  "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n",
204  m_name, mem_before, mem_after);
205  }
206  return true;
207 }
208 
210  std::string memory;
211  if (!pdb->GetProperty("leveldb.approximate-memory-usage", &memory)) {
213  "Failed to get approximate-memory-usage property\n");
214  return 0;
215  }
216  return stoul(memory);
217 }
218 
219 // Prefixed with null character to avoid collisions with other keys
220 //
221 // We must use a string constructor which specifies length so that we copy past
222 // the null-terminator.
223 const std::string CDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14);
224 
225 const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8;
226 
231 std::vector<uint8_t> CDBWrapper::CreateObfuscateKey() const {
232  uint8_t buff[OBFUSCATE_KEY_NUM_BYTES];
234  return std::vector<uint8_t>(&buff[0], &buff[OBFUSCATE_KEY_NUM_BYTES]);
235 }
236 
238  std::unique_ptr<CDBIterator> it(NewIterator());
239  it->SeekToFirst();
240  return !(it->Valid());
241 }
242 
244  delete piter;
245 }
246 bool CDBIterator::Valid() const {
247  return piter->Valid();
248 }
250  piter->SeekToFirst();
251 }
253  piter->Next();
254 }
255 
256 namespace dbwrapper_private {
257 
258 void HandleError(const leveldb::Status &status) {
259  if (status.ok()) {
260  return;
261  }
262  const std::string errmsg = "Fatal LevelDB error: " + status.ToString();
263  LogPrintf("%s\n", errmsg);
264  LogPrintf("You can use -debug=leveldb to get more complete diagnostic "
265  "messages\n");
266  throw dbwrapper_error(errmsg);
267 }
268 
269 const std::vector<uint8_t> &GetObfuscateKey(const CDBWrapper &w) {
270  return w.obfuscate_key;
271 }
272 }; // namespace dbwrapper_private
ArgsManager::GetBoolArg
bool GetBoolArg(const std::string &strArg, bool fDefault) const
Return boolean argument or default value.
Definition: system.cpp:601
dbwrapper.h
tinyformat::format
void format(std::ostream &out, const char *fmt, const Args &... args)
Format list of arguments to the stream according to given format string.
Definition: tinyformat.h:1111
CDBBatch
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:48
CDBWrapper::obfuscate_key
std::vector< uint8_t > obfuscate_key
a key used for optional XOR-obfuscation of the database
Definition: dbwrapper.h:202
LogPrintfToBeContinued
#define LogPrintfToBeContinued
These are aliases used to explicitly state that the message should not end with a newline character.
Definition: logging.h:205
CDBWrapper::CreateObfuscateKey
std::vector< uint8_t > CreateObfuscateKey() const
Returns a string (consisting of 8 random bytes) suitable for use as an obfuscating XOR key.
Definition: dbwrapper.cpp:231
GetRandBytes
void GetRandBytes(uint8_t *buf, int num) noexcept
Overall design of the RNG and entropy sources.
Definition: random.cpp:634
fs::PathToString
static std::string PathToString(const path &path)
Convert path object to byte string.
Definition: fs.h:134
CBitcoinLevelDBLogger
Definition: dbwrapper.cpp:18
CDBWrapper::syncoptions
leveldb::WriteOptions syncoptions
options used when sync writing to the database
Definition: dbwrapper.h:193
SetMaxOpenFiles
static void SetMaxOpenFiles(leveldb::Options *options)
Definition: dbwrapper.cpp:76
CDBWrapper::CDBWrapper
CDBWrapper(const fs::path &path, size_t nCacheSize, bool fMemory=false, bool fWipe=false, bool obfuscate=false)
Definition: dbwrapper.cpp:119
GetOptions
static leveldb::Options GetOptions(size_t nCacheSize)
Definition: dbwrapper.cpp:101
CDBIterator::Next
void Next()
Definition: dbwrapper.cpp:252
random.h
CDBWrapper::NewIterator
CDBIterator * NewIterator()
Definition: dbwrapper.h:289
fs::path
Path class wrapper to prepare application code for transition from boost::filesystem library to std::...
Definition: fs.h:33
BCLog::LEVELDB
@ LEVELDB
Definition: logging.h:58
CDBWrapper::penv
leveldb::Env * penv
custom environment this database is using (may be nullptr in case of default environment)
Definition: dbwrapper.h:178
CDBWrapper::OBFUSCATE_KEY_KEY
static const std::string OBFUSCATE_KEY_KEY
the key under which the obfuscation key is stored
Definition: dbwrapper.h:205
CDBIterator::~CDBIterator
~CDBIterator()
Definition: dbwrapper.cpp:243
dbwrapper_private::GetObfuscateKey
const std::vector< uint8_t > & GetObfuscateKey(const CDBWrapper &w)
Work around circular dependency, as well as for testing in dbwrapper_tests.
Definition: dbwrapper.cpp:269
CDBBatch::batch
leveldb::WriteBatch batch
Definition: dbwrapper.h:53
CDBWrapper::pdb
leveldb::DB * pdb
the database itself
Definition: dbwrapper.h:196
dbwrapper_private
These should be considered an implementation detail of the specific database.
Definition: dbwrapper.cpp:256
CDBIterator::piter
leveldb::Iterator * piter
Definition: dbwrapper.h:118
LogPrint
#define LogPrint(category,...)
Definition: logging.h:193
CDBWrapper::DynamicMemoryUsage
size_t DynamicMemoryUsage() const
Definition: dbwrapper.cpp:209
CDBWrapper::writeoptions
leveldb::WriteOptions writeoptions
options used when writing to the database
Definition: dbwrapper.h:190
CDBWrapper::m_name
std::string m_name
the name of this database
Definition: dbwrapper.h:199
dbwrapper_private::HandleError
void HandleError(const leveldb::Status &status)
Handle database error by throwing dbwrapper_error exception.
Definition: dbwrapper.cpp:258
CDBWrapper
Definition: dbwrapper.h:171
CDBIterator::Valid
bool Valid() const
Definition: dbwrapper.cpp:246
dbwrapper_error
Definition: dbwrapper.h:21
LogAcceptCategory
static bool LogAcceptCategory(BCLog::LogFlags category)
Return true if log accepts specified category.
Definition: logging.h:164
CDBWrapper::options
leveldb::Options options
database options used
Definition: dbwrapper.h:181
gArgs
ArgsManager gArgs
Definition: system.cpp:76
CDBWrapper::~CDBWrapper
~CDBWrapper()
Definition: dbwrapper.cpp:177
CDBWrapper::OBFUSCATE_KEY_NUM_BYTES
static const unsigned int OBFUSCATE_KEY_NUM_BYTES
the length of the obfuscate key in number of bytes
Definition: dbwrapper.h:208
CDBIterator::SeekToFirst
void SeekToFirst()
Definition: dbwrapper.cpp:249
CDBWrapper::IsEmpty
bool IsEmpty()
Return true if the database managed by this class contains no entries.
Definition: dbwrapper.cpp:237
TryCreateDirectories
bool TryCreateDirectories(const fs::path &p)
Ignores exceptions thrown by Boost's create_directories if the requested directory exists.
Definition: system.cpp:1135
CBitcoinLevelDBLogger::Logv
void Logv(const char *format, va_list ap) override
Definition: dbwrapper.cpp:23
HexStr
std::string HexStr(const Span< const uint8_t > s)
Convert a span of bytes to a lower-case hexadecimal string.
Definition: strencodings.cpp:656
CDBWrapper::WriteBatch
bool WriteBatch(CDBBatch &batch, bool fSync=false)
Definition: dbwrapper.cpp:190
LogPrintf
static void LogPrintf(const char *fmt, const Args &... args)
Definition: logging.h:175