Bitcoin ABC  0.24.10
P2P Digital Currency
lockedpool.cpp
Go to the documentation of this file.
1 // Copyright (c) 2016 The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 
5 #include <support/cleanse.h>
6 #include <support/lockedpool.h>
7 
8 #if defined(HAVE_CONFIG_H)
9 #include <config/bitcoin-config.h>
10 #endif
11 
12 #ifdef WIN32
13 #ifndef NOMINMAX
14 #define NOMINMAX
15 #endif
16 #include <windows.h>
17 #else
18 #include <climits> // for PAGESIZE
19 #include <sys/mman.h> // for mmap
20 #include <sys/resource.h> // for getrlimit
21 #include <unistd.h> // for sysconf
22 #endif
23 
24 #include <algorithm>
25 #ifdef ARENA_DEBUG
26 #include <iomanip>
27 #include <iostream>
28 #endif
29 
31 
32 /*******************************************************************************/
33 // Utilities
34 //
36 static inline size_t align_up(size_t x, size_t align) {
37  return (x + align - 1) & ~(align - 1);
38 }
39 
40 /*******************************************************************************/
41 // Implementation: Arena
42 
43 Arena::Arena(void *base_in, size_t size_in, size_t alignment_in)
44  : base(static_cast<char *>(base_in)),
45  end(static_cast<char *>(base_in) + size_in), alignment(alignment_in) {
46  // Start with one free chunk that covers the entire arena
47  auto it = size_to_free_chunk.emplace(size_in, base);
48  chunks_free.emplace(base, it);
49  chunks_free_end.emplace(base + size_in, it);
50 }
51 
53 
54 void *Arena::alloc(size_t size) {
55  // Round to next multiple of alignment
56  size = align_up(size, alignment);
57 
58  // Don't handle zero-sized chunks
59  if (size == 0) {
60  return nullptr;
61  }
62 
63  // Pick a large enough free-chunk. Returns an iterator pointing to the first
64  // element that is not less than key. This allocation strategy is best-fit.
65  // According to "Dynamic Storage Allocation: A Survey and Critical Review",
66  // Wilson et. al. 1995,
67  // http://www.scs.stanford.edu/14wi-cs140/sched/readings/wilson.pdf,
68  // best-fit and first-fit policies seem to work well in practice.
69  auto size_ptr_it = size_to_free_chunk.lower_bound(size);
70  if (size_ptr_it == size_to_free_chunk.end()) {
71  return nullptr;
72  }
73 
74  // Create the used-chunk, taking its space from the end of the free-chunk
75  const size_t size_remaining = size_ptr_it->first - size;
76  auto alloced =
77  chunks_used.emplace(size_ptr_it->second + size_remaining, size).first;
78  chunks_free_end.erase(size_ptr_it->second + size_ptr_it->first);
79  if (size_ptr_it->first == size) {
80  // whole chunk is used up
81  chunks_free.erase(size_ptr_it->second);
82  } else {
83  // still some memory left in the chunk
84  auto it_remaining =
85  size_to_free_chunk.emplace(size_remaining, size_ptr_it->second);
86  chunks_free[size_ptr_it->second] = it_remaining;
87  chunks_free_end.emplace(size_ptr_it->second + size_remaining,
88  it_remaining);
89  }
90  size_to_free_chunk.erase(size_ptr_it);
91 
92  return reinterpret_cast<void *>(alloced->first);
93 }
94 
95 void Arena::free(void *ptr) {
96  // Freeing the nullptr pointer is OK.
97  if (ptr == nullptr) {
98  return;
99  }
100 
101  // Remove chunk from used map
102  auto i = chunks_used.find(static_cast<char *>(ptr));
103  if (i == chunks_used.end()) {
104  throw std::runtime_error("Arena: invalid or double free");
105  }
106  std::pair<char *, size_t> freed = *i;
107  chunks_used.erase(i);
108 
109  // coalesce freed with previous chunk
110  auto prev = chunks_free_end.find(freed.first);
111  if (prev != chunks_free_end.end()) {
112  freed.first -= prev->second->first;
113  freed.second += prev->second->first;
114  size_to_free_chunk.erase(prev->second);
115  chunks_free_end.erase(prev);
116  }
117 
118  // coalesce freed with chunk after freed
119  auto next = chunks_free.find(freed.first + freed.second);
120  if (next != chunks_free.end()) {
121  freed.second += next->second->first;
122  size_to_free_chunk.erase(next->second);
123  chunks_free.erase(next);
124  }
125 
126  // Add/set space with coalesced free chunk
127  auto it = size_to_free_chunk.emplace(freed.second, freed.first);
128  chunks_free[freed.first] = it;
129  chunks_free_end[freed.first + freed.second] = it;
130 }
131 
133  Arena::Stats r{0, 0, 0, chunks_used.size(), chunks_free.size()};
134  for (const auto &chunk : chunks_used) {
135  r.used += chunk.second;
136  }
137  for (const auto &chunk : chunks_free) {
138  r.free += chunk.second->first;
139  }
140  r.total = r.used + r.free;
141  return r;
142 }
143 
144 #ifdef ARENA_DEBUG
145 static void printchunk(void *base, size_t sz, bool used) {
146  std::cout << "0x" << std::hex << std::setw(16) << std::setfill('0') << base
147  << " 0x" << std::hex << std::setw(16) << std::setfill('0') << sz
148  << " 0x" << used << std::endl;
149 }
150 void Arena::walk() const {
151  for (const auto &chunk : chunks_used) {
152  printchunk(chunk.first, chunk.second, true);
153  }
154  std::cout << std::endl;
155  for (const auto &chunk : chunks_free) {
156  printchunk(chunk.first, chunk.second->first, false);
157  }
158  std::cout << std::endl;
159 }
160 #endif
161 
162 /*******************************************************************************/
163 // Implementation: Win32LockedPageAllocator
164 
165 #ifdef WIN32
166 
169 class Win32LockedPageAllocator : public LockedPageAllocator {
170 public:
171  Win32LockedPageAllocator();
172  void *AllocateLocked(size_t len, bool *lockingSuccess) override;
173  void FreeLocked(void *addr, size_t len) override;
174  size_t GetLimit() override;
175 
176 private:
177  size_t page_size;
178 };
179 
180 Win32LockedPageAllocator::Win32LockedPageAllocator() {
181  // Determine system page size in bytes
182  SYSTEM_INFO sSysInfo;
183  GetSystemInfo(&sSysInfo);
184  page_size = sSysInfo.dwPageSize;
185 }
186 void *Win32LockedPageAllocator::AllocateLocked(size_t len,
187  bool *lockingSuccess) {
188  len = align_up(len, page_size);
189  void *addr =
190  VirtualAlloc(nullptr, len, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
191  if (addr) {
192  // VirtualLock is used to attempt to keep keying material out of swap.
193  // Note that it does not provide this as a guarantee, but, in practice,
194  // memory that has been VirtualLock'd almost never gets written to the
195  // pagefile except in rare circumstances where memory is extremely low.
196  *lockingSuccess = VirtualLock(const_cast<void *>(addr), len) != 0;
197  }
198  return addr;
199 }
200 void Win32LockedPageAllocator::FreeLocked(void *addr, size_t len) {
201  len = align_up(len, page_size);
202  memory_cleanse(addr, len);
203  VirtualUnlock(const_cast<void *>(addr), len);
204 }
205 
206 size_t Win32LockedPageAllocator::GetLimit() {
207  // TODO is there a limit on Windows, how to get it?
208  return std::numeric_limits<size_t>::max();
209 }
210 #endif
211 
212 /*******************************************************************************/
213 // Implementation: PosixLockedPageAllocator
214 
215 #ifndef WIN32
216 
221 public:
223  void *AllocateLocked(size_t len, bool *lockingSuccess) override;
224  void FreeLocked(void *addr, size_t len) override;
225  size_t GetLimit() override;
226 
227 private:
228  size_t page_size;
229 };
230 
232 // Determine system page size in bytes
233 #if defined(PAGESIZE) // defined in climits
234  page_size = PAGESIZE;
235 #else // assume some POSIX OS
236  page_size = sysconf(_SC_PAGESIZE);
237 #endif
238 }
239 
240 // Some systems (at least OS X) do not define MAP_ANONYMOUS yet and define
241 // MAP_ANON which is deprecated
242 #ifndef MAP_ANONYMOUS
243 #define MAP_ANONYMOUS MAP_ANON
244 #endif
245 
247  bool *lockingSuccess) {
248  void *addr;
249  len = align_up(len, page_size);
250  addr = mmap(nullptr, len, PROT_READ | PROT_WRITE,
251  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
252  if (addr == MAP_FAILED) {
253  return nullptr;
254  }
255  if (addr) {
256  *lockingSuccess = mlock(addr, len) == 0;
257 #if defined(MADV_DONTDUMP) // Linux
258  madvise(addr, len, MADV_DONTDUMP);
259 #elif defined(MADV_NOCORE) // FreeBSD
260  madvise(addr, len, MADV_NOCORE);
261 #endif
262  }
263  return addr;
264 }
265 void PosixLockedPageAllocator::FreeLocked(void *addr, size_t len) {
266  len = align_up(len, page_size);
267  memory_cleanse(addr, len);
268  munlock(addr, len);
269  munmap(addr, len);
270 }
272 #ifdef RLIMIT_MEMLOCK
273  struct rlimit rlim;
274  if (getrlimit(RLIMIT_MEMLOCK, &rlim) == 0) {
275  if (rlim.rlim_cur != RLIM_INFINITY) {
276  return rlim.rlim_cur;
277  }
278  }
279 #endif
280  return std::numeric_limits<size_t>::max();
281 }
282 #endif
283 
284 /*******************************************************************************/
285 // Implementation: LockedPool
286 
287 LockedPool::LockedPool(std::unique_ptr<LockedPageAllocator> allocator_in,
288  LockingFailed_Callback lf_cb_in)
289  : allocator(std::move(allocator_in)), lf_cb(lf_cb_in),
290  cumulative_bytes_locked(0) {}
291 
293 void *LockedPool::alloc(size_t size) {
294  std::lock_guard<std::mutex> lock(mutex);
295 
296  // Don't handle impossible sizes
297  if (size == 0 || size > ARENA_SIZE) {
298  return nullptr;
299  }
300 
301  // Try allocating from each current arena
302  for (auto &arena : arenas) {
303  void *addr = arena.alloc(size);
304  if (addr) {
305  return addr;
306  }
307  }
308  // If that fails, create a new one
310  return arenas.back().alloc(size);
311  }
312  return nullptr;
313 }
314 
315 void LockedPool::free(void *ptr) {
316  std::lock_guard<std::mutex> lock(mutex);
317  // TODO we can do better than this linear search by keeping a map of arena
318  // extents to arena, and looking up the address.
319  for (auto &arena : arenas) {
320  if (arena.addressInArena(ptr)) {
321  arena.free(ptr);
322  return;
323  }
324  }
325  throw std::runtime_error(
326  "LockedPool: invalid address not pointing to any arena");
327 }
328 
330  std::lock_guard<std::mutex> lock(mutex);
331  LockedPool::Stats r{0, 0, 0, cumulative_bytes_locked, 0, 0};
332  for (const auto &arena : arenas) {
333  Arena::Stats i = arena.stats();
334  r.used += i.used;
335  r.free += i.free;
336  r.total += i.total;
337  r.chunks_used += i.chunks_used;
338  r.chunks_free += i.chunks_free;
339  }
340  return r;
341 }
342 
343 bool LockedPool::new_arena(size_t size, size_t align) {
344  bool locked;
345  // If this is the first arena, handle this specially: Cap the upper size by
346  // the process limit. This makes sure that the first arena will at least be
347  // locked. An exception to this is if the process limit is 0: in this case
348  // no memory can be locked at all so we'll skip past this logic.
349  if (arenas.empty()) {
350  size_t limit = allocator->GetLimit();
351  if (limit > 0) {
352  size = std::min(size, limit);
353  }
354  }
355  void *addr = allocator->AllocateLocked(size, &locked);
356  if (!addr) {
357  return false;
358  }
359  if (locked) {
360  cumulative_bytes_locked += size;
361  } else if (lf_cb) {
362  // Call the locking-failed callback if locking failed
363  if (!lf_cb()) {
364  // If the callback returns false, free the memory and fail,
365  // otherwise consider the user warned and proceed.
366  allocator->FreeLocked(addr, size);
367  return false;
368  }
369  }
370  arenas.emplace_back(allocator.get(), addr, size, align);
371  return true;
372 }
373 
375  void *base_in, size_t size_in,
376  size_t align_in)
377  : Arena(base_in, size_in, align_in), base(base_in), size(size_in),
378  allocator(allocator_in) {}
380  allocator->FreeLocked(base, size);
381 }
382 
383 /*******************************************************************************/
384 // Implementation: LockedPoolManager
385 //
387  std::unique_ptr<LockedPageAllocator> allocator_in)
388  : LockedPool(std::move(allocator_in), &LockedPoolManager::LockingFailed) {}
389 
391  // TODO: log something but how? without including util.h
392  return true;
393 }
394 
396 // Using a local static instance guarantees that the object is initialized when
397 // it's first needed and also deinitialized after all objects that use it are
398 // done with it. I can think of one unlikely scenario where we may have a static
399 // deinitialization order/problem, but the check in LockedPoolManagerBase's
400 // destructor helps us detect if that ever happens.
401 #ifdef WIN32
402  std::unique_ptr<LockedPageAllocator> allocator(
403  new Win32LockedPageAllocator());
404 #else
405  std::unique_ptr<LockedPageAllocator> allocator(
407 #endif
408  static LockedPoolManager instance(std::move(allocator));
409  LockedPoolManager::_instance = &instance;
410 }
PosixLockedPageAllocator::page_size
size_t page_size
Definition: lockedpool.cpp:228
LockedPoolManager
Singleton class to keep track of locked (ie, non-swappable) memory, for use in std::allocator templat...
Definition: lockedpool.h:233
LockedPool::new_arena
bool new_arena(size_t size, size_t align)
Definition: lockedpool.cpp:343
LockedPool::ARENA_ALIGN
static const size_t ARENA_ALIGN
Chunk alignment.
Definition: lockedpool.h:145
PosixLockedPageAllocator::GetLimit
size_t GetLimit() override
Get the total limit on the amount of memory that may be locked by this process, in bytes.
Definition: lockedpool.cpp:271
LockedPool::LockedPageArena::LockedPageArena
LockedPageArena(LockedPageAllocator *alloc_in, void *base_in, size_t size, size_t align)
Definition: lockedpool.cpp:374
Arena::Stats::chunks_free
size_t chunks_free
Definition: lockedpool.h:64
Arena::Stats::chunks_used
size_t chunks_used
Definition: lockedpool.h:63
align_up
static size_t align_up(size_t x, size_t align)
Align up to power of 2.
Definition: lockedpool.cpp:36
LockedPool::allocator
std::unique_ptr< LockedPageAllocator > allocator
Definition: lockedpool.h:196
MAP_ANONYMOUS
#define MAP_ANONYMOUS
Definition: lockedpool.cpp:243
memory_cleanse
void memory_cleanse(void *ptr, size_t len)
Secure overwrite a buffer (possibly containing secret data) with zero-bytes.
Definition: cleanse.cpp:14
LockedPool::lf_cb
LockingFailed_Callback lf_cb
Definition: lockedpool.h:214
Arena::Stats::total
size_t total
Definition: lockedpool.h:62
PosixLockedPageAllocator
LockedPageAllocator specialized for OSes that don't try to be special snowflakes.
Definition: lockedpool.cpp:220
Arena::chunks_used
std::unordered_map< char *, size_t > chunks_used
Map from begin of used chunk to its size.
Definition: lockedpool.h:108
PosixLockedPageAllocator::PosixLockedPageAllocator
PosixLockedPageAllocator()
Definition: lockedpool.cpp:231
LockedPageAllocator::FreeLocked
virtual void FreeLocked(void *addr, size_t len)=0
Unlock and free memory pages.
Arena::chunks_free_end
ChunkToSizeMap chunks_free_end
Map from end of free chunk to its node in size_to_free_chunk.
Definition: lockedpool.h:105
Arena::base
char * base
Base address of arena.
Definition: lockedpool.h:111
LockedPool::arenas
std::list< LockedPageArena > arenas
Definition: lockedpool.h:213
Arena::~Arena
virtual ~Arena()
Definition: lockedpool.cpp:52
LockedPool::alloc
void * alloc(size_t size)
Allocate size bytes from this arena.
Definition: lockedpool.cpp:293
cleanse.h
Arena::stats
Stats stats() const
Get arena usage statistics.
Definition: lockedpool.cpp:132
Arena::alignment
size_t alignment
Minimum chunk alignment.
Definition: lockedpool.h:115
Arena::Stats::free
size_t free
Definition: lockedpool.h:61
Arena::size_to_free_chunk
SizeToChunkSortedMap size_to_free_chunk
Map to enable O(log(n)) best-fit allocation, as it's sorted by size.
Definition: lockedpool.h:98
LockedPool::Stats
Memory statistics.
Definition: lockedpool.h:153
Arena::Stats
Memory statistics.
Definition: lockedpool.h:59
Arena::chunks_free
ChunkToSizeMap chunks_free
Map from begin of free chunk to its node in size_to_free_chunk.
Definition: lockedpool.h:103
LockedPoolManager::_instance
static LockedPoolManager * _instance
Definition: lockedpool.h:250
LockedPool::~LockedPool
~LockedPool()
Definition: lockedpool.cpp:292
PosixLockedPageAllocator::AllocateLocked
void * AllocateLocked(size_t len, bool *lockingSuccess) override
Allocate and lock memory pages.
Definition: lockedpool.cpp:246
Arena
An arena manages a contiguous region of memory by dividing it into chunks.
Definition: lockedpool.h:50
LockedPoolManager::LockingFailed
static bool LockingFailed()
Called when locking fails, warn the user here.
Definition: lockedpool.cpp:390
Arena::free
void free(void *ptr)
Free a previously allocated chunk of memory.
Definition: lockedpool.cpp:95
LockedPool::free
void free(void *ptr)
Free a previously allocated chunk of memory.
Definition: lockedpool.cpp:315
LockedPageAllocator::AllocateLocked
virtual void * AllocateLocked(size_t len, bool *lockingSuccess)=0
Allocate and lock memory pages.
Arena::Arena
Arena(void *base, size_t size, size_t alignment)
Definition: lockedpool.cpp:43
lockedpool.h
LockedPool::ARENA_SIZE
static const size_t ARENA_SIZE
Size of one arena of locked memory.
Definition: lockedpool.h:140
LockedPool
Pool for locked memory chunks.
Definition: lockedpool.h:132
Arena::alloc
void * alloc(size_t size)
Allocate size bytes from this arena.
Definition: lockedpool.cpp:54
LockedPoolManager::CreateInstance
static void CreateInstance()
Create a new LockedPoolManager specialized to the OS.
Definition: lockedpool.cpp:395
LockedPageAllocator::GetLimit
virtual size_t GetLimit()=0
Get the total limit on the amount of memory that may be locked by this process, in bytes.
LockedPool::stats
Stats stats() const
Get pool usage statistics.
Definition: lockedpool.cpp:329
Arena::Stats::used
size_t used
Definition: lockedpool.h:60
LockedPool::LockedPool
LockedPool(std::unique_ptr< LockedPageAllocator > allocator, LockingFailed_Callback lf_cb_in=nullptr)
Create a new LockedPool.
Definition: lockedpool.cpp:287
LockedPool::cumulative_bytes_locked
size_t cumulative_bytes_locked
Definition: lockedpool.h:215
LockedPoolManager::LockedPoolManager
LockedPoolManager(std::unique_ptr< LockedPageAllocator > allocator)
Definition: lockedpool.cpp:386
LockedPageAllocator
OS-dependent allocation and deallocation of locked/pinned memory pages.
Definition: lockedpool.h:19
PosixLockedPageAllocator::FreeLocked
void FreeLocked(void *addr, size_t len) override
Unlock and free memory pages.
Definition: lockedpool.cpp:265
LockedPool::LockedPageArena::~LockedPageArena
~LockedPageArena()
Definition: lockedpool.cpp:379
LockedPool::mutex
std::mutex mutex
Mutex protects access to this pool's data structures, including arenas.
Definition: lockedpool.h:219