--- a/memory/build/mozjemalloc.cpp
+++ b/memory/build/mozjemalloc.cpp
@@ -366,20 +366,16 @@ struct arena_chunk_t
// Map of pages within chunk that keeps track of free/large/small.
arena_chunk_map_t map[1]; // Dynamically sized.
};
// ***************************************************************************
// Constants defining allocator size classes and behavior.
-// Size and alignment of memory chunks that are allocated by the OS's virtual
-// memory system.
-#define CHUNK_2POW_DEFAULT 20
-
// Maximum size of L1 cache line. This is used to avoid cache line aliasing,
// so over-estimates are okay (up to a point), but under-estimates will
// negatively affect performance.
static const size_t kCacheLineSize = 64;
// Smallest size class to support. On Windows the smallest allocation size
// must be 8 bytes on 32-bit, 16 bytes on 64-bit. On Linux and Mac, even
// malloc(1) must reserve a word's worth of memory (see Mozilla bug 691003).
@@ -410,19 +406,20 @@ static_assert(kMaxQuantumClass % kQuantu
// Number of (2^n)-spaced tiny bins.
static const unsigned ntbins =
unsigned(LOG2(kMinQuantumClass) - LOG2(kMinTinyClass));
// Number of quantum-spaced bins.
static const unsigned nqbins = unsigned(kMaxQuantumClass / kQuantum);
-#define CHUNKSIZE_DEFAULT ((size_t)1 << CHUNK_2POW_DEFAULT)
-static const size_t chunksize = CHUNKSIZE_DEFAULT;
-static const size_t chunksize_mask = CHUNKSIZE_DEFAULT - 1;
+// Size and alignment of memory chunks that are allocated by the OS's virtual
+// memory system.
+static const size_t kChunkSize = 1_MiB;
+static const size_t kChunkSizeMask = kChunkSize - 1;
#ifdef MALLOC_STATIC_PAGESIZE
// VM page size. It must divide the runtime CPU page size or the code
// will abort.
// Platform specific page size conditions copied from js/public/HeapAPI.h
#if (defined(SOLARIS) || defined(__FreeBSD__)) && \
(defined(__sparc) || defined(__sparcv9) || defined(__ia64))
static const size_t pagesize = 8_KiB;
@@ -477,44 +474,42 @@ DEFINE_GLOBAL(size_t) gMaxSubPageClass =
// Number of (2^n)-spaced sub-page bins.
DEFINE_GLOBAL(uint8_t)
nsbins = GLOBAL_LOG2(gMaxSubPageClass) - LOG2(kMaxQuantumClass);
DEFINE_GLOBAL(uint8_t) pagesize_2pow = GLOBAL_LOG2(pagesize);
DEFINE_GLOBAL(size_t) pagesize_mask = pagesize - 1;
// Number of pages in a chunk.
-DEFINE_GLOBAL(size_t) chunk_npages = chunksize >> pagesize_2pow;
+DEFINE_GLOBAL(size_t) chunk_npages = kChunkSize >> pagesize_2pow;
// Number of pages necessary for a chunk header.
DEFINE_GLOBAL(size_t)
arena_chunk_header_npages =
((sizeof(arena_chunk_t) + sizeof(arena_chunk_map_t) * (chunk_npages - 1) +
pagesize_mask) &
~pagesize_mask) >>
pagesize_2pow;
// Max size class for arenas.
DEFINE_GLOBAL(size_t)
-gMaxLargeClass = chunksize - (arena_chunk_header_npages << pagesize_2pow);
+gMaxLargeClass = kChunkSize - (arena_chunk_header_npages << pagesize_2pow);
// Various sanity checks that regard configuration.
GLOBAL_ASSERT(1ULL << pagesize_2pow == pagesize,
"Page size is not a power of two");
GLOBAL_ASSERT(kQuantum >= sizeof(void*));
GLOBAL_ASSERT(kQuantum <= pagesize);
-GLOBAL_ASSERT(chunksize >= pagesize);
-GLOBAL_ASSERT(kQuantum * 4 <= chunksize);
+GLOBAL_ASSERT(kChunkSize >= pagesize);
+GLOBAL_ASSERT(kQuantum * 4 <= kChunkSize);
END_GLOBALS
-// Recycle at most 128 chunks. With 1 MiB chunks, this means we retain at most
+// Recycle at most 128 MiB of chunks. This means we retain at most
// 6.25% of the process address space on a 32-bit OS for later use.
-#define CHUNK_RECYCLE_LIMIT 128
-
-static const size_t gRecycleLimit = CHUNK_RECYCLE_LIMIT * CHUNKSIZE_DEFAULT;
+static const size_t gRecycleLimit = 128_MiB;
// The current amount of recycled bytes, updated atomically.
static Atomic<size_t, ReleaseAcquire> gRecycledSize;
// Maximum number of dirty pages per arena.
#define DIRTY_MAX_DEFAULT (1U << 8)
static size_t opt_dirty_max = DIRTY_MAX_DEFAULT;
@@ -534,17 +529,17 @@ static size_t opt_dirty_max = DIRTY_MAX_
//
// (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP))
#define RUN_BFP 12
// \/ Implicit binary fixed point.
#define RUN_MAX_OVRHD 0x0000003dU
#define RUN_MAX_OVRHD_RELAX 0x00001800U
// Return the smallest chunk multiple that is >= s.
-#define CHUNK_CEILING(s) (((s) + chunksize_mask) & ~chunksize_mask)
+#define CHUNK_CEILING(s) (((s) + kChunkSizeMask) & ~kChunkSizeMask)
// Return the smallest cacheline multiple that is >= s.
#define CACHELINE_CEILING(s) \
(((s) + (kCacheLineSize - 1)) & ~(kCacheLineSize - 1))
// Return the smallest quantum multiple that is >= a.
#define QUANTUM_CEILING(a) (((a) + (kQuantumMask)) & ~(kQuantumMask))
@@ -1159,17 +1154,17 @@ private:
Tree mArenas;
Tree mPrivateArenas;
};
static ArenaCollection gArenas;
// ******
// Chunks.
-static AddressRadixTree<(sizeof(void*) << 3) - CHUNK_2POW_DEFAULT> gChunkRTree;
+static AddressRadixTree<(sizeof(void*) << 3) - LOG2(kChunkSize)> gChunkRTree;
// Protects chunk-related data structures.
static Mutex chunks_mtx;
// Trees of chunks that were previously allocated (trees differ only in node
// ordering). These are used when allocating chunks, in an attempt to re-use
// address space. Depending on function, different tree orderings are needed,
// which is why there are two trees with the same contents.
@@ -1381,24 +1376,24 @@ Mutex::Unlock()
// End mutex.
// ***************************************************************************
// Begin Utility functions/macros.
// Return the chunk address for allocation address a.
static inline arena_chunk_t*
GetChunkForPtr(const void* aPtr)
{
- return (arena_chunk_t*)(uintptr_t(aPtr) & ~chunksize_mask);
+ return (arena_chunk_t*)(uintptr_t(aPtr) & ~kChunkSizeMask);
}
// Return the chunk offset of address a.
static inline size_t
GetChunkOffsetForPtr(const void* aPtr)
{
- return (size_t)(uintptr_t(aPtr) & chunksize_mask);
+ return (size_t)(uintptr_t(aPtr) & kChunkSizeMask);
}
static inline const char*
_getprogname(void)
{
return "<jemalloc>";
}
@@ -1408,24 +1403,24 @@ static inline const char*
static inline void
pages_decommit(void* aAddr, size_t aSize)
{
#ifdef XP_WIN
// The region starting at addr may have been allocated in multiple calls
// to VirtualAlloc and recycled, so decommitting the entire region in one
// go may not be valid. However, since we allocate at least a chunk at a
// time, we may touch any region in chunksized increments.
- size_t pages_size = std::min(aSize, chunksize - GetChunkOffsetForPtr(aAddr));
+ size_t pages_size = std::min(aSize, kChunkSize - GetChunkOffsetForPtr(aAddr));
while (aSize > 0) {
if (!VirtualFree(aAddr, pages_size, MEM_DECOMMIT)) {
MOZ_CRASH();
}
aAddr = (void*)((uintptr_t)aAddr + pages_size);
aSize -= pages_size;
- pages_size = std::min(aSize, chunksize);
+ pages_size = std::min(aSize, kChunkSize);
}
#else
if (mmap(
aAddr, aSize, PROT_NONE, MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1, 0) ==
MAP_FAILED) {
MOZ_CRASH();
}
MozTagAnonymousMemory(aAddr, aSize, "jemalloc-decommitted");
@@ -1436,24 +1431,24 @@ pages_decommit(void* aAddr, size_t aSize
MOZ_MUST_USE static inline bool
pages_commit(void* aAddr, size_t aSize)
{
#ifdef XP_WIN
// The region starting at addr may have been allocated in multiple calls
// to VirtualAlloc and recycled, so committing the entire region in one
// go may not be valid. However, since we allocate at least a chunk at a
// time, we may touch any region in chunksized increments.
- size_t pages_size = std::min(aSize, chunksize - GetChunkOffsetForPtr(aAddr));
+ size_t pages_size = std::min(aSize, kChunkSize - GetChunkOffsetForPtr(aAddr));
while (aSize > 0) {
if (!VirtualAlloc(aAddr, pages_size, MEM_COMMIT, PAGE_READWRITE)) {
return false;
}
aAddr = (void*)((uintptr_t)aAddr + pages_size);
aSize -= pages_size;
- pages_size = std::min(aSize, chunksize);
+ pages_size = std::min(aSize, kChunkSize);
}
#else
if (mmap(aAddr,
aSize,
PROT_READ | PROT_WRITE,
MAP_FIXED | MAP_PRIVATE | MAP_ANON,
-1,
0) == MAP_FAILED) {
@@ -1467,17 +1462,17 @@ pages_commit(void* aAddr, size_t aSize)
static bool
base_pages_alloc(size_t minsize)
{
size_t csize;
size_t pminsize;
MOZ_ASSERT(minsize != 0);
csize = CHUNK_CEILING(minsize);
- base_pages = chunk_alloc(csize, chunksize, true);
+ base_pages = chunk_alloc(csize, kChunkSize, true);
if (!base_pages) {
return true;
}
base_next_addr = base_pages;
base_past_addr = (void*)((uintptr_t)base_pages + csize);
// Leave enough pages for minsize committed, since otherwise they would
// have to be immediately recommitted.
pminsize = PAGE_CEILING(minsize);
@@ -1637,17 +1632,17 @@ pages_map(void* aAddr, size_t aSize)
#if defined(__sparc__) && defined(__arch64__) && defined(__linux__)
const uintptr_t start = 0x0000070000000000ULL;
const uintptr_t end = 0x0000800000000000ULL;
// Copied from js/src/gc/Memory.cpp and adapted for this source
uintptr_t hint;
void* region = MAP_FAILED;
for (hint = start; region == MAP_FAILED && hint + aSize <= end;
- hint += chunksize) {
+ hint += kChunkSize) {
region = mmap((void*)hint,
aSize,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANON,
-1,
0);
if (region != MAP_FAILED) {
if (((size_t)region + (aSize - 1)) & 0xffff800000000000) {
@@ -1925,22 +1920,22 @@ pages_purge(void* addr, size_t length, b
memset(addr, 0, length);
}
#endif
#ifdef XP_WIN
// The region starting at addr may have been allocated in multiple calls
// to VirtualAlloc and recycled, so resetting the entire region in one
// go may not be valid. However, since we allocate at least a chunk at a
// time, we may touch any region in chunksized increments.
- size_t pages_size = std::min(length, chunksize - GetChunkOffsetForPtr(addr));
+ size_t pages_size = std::min(length, kChunkSize - GetChunkOffsetForPtr(addr));
while (length > 0) {
VirtualAlloc(addr, pages_size, MEM_RESET, PAGE_READWRITE);
addr = (void*)((uintptr_t)addr + pages_size);
length -= pages_size;
- pages_size = std::min(length, chunksize);
+ pages_size = std::min(length, kChunkSize);
}
return force_zero;
#else
#ifdef XP_LINUX
#define JEMALLOC_MADV_PURGE MADV_DONTNEED
#define JEMALLOC_MADV_ZEROS true
#else // FreeBSD and Darwin.
#define JEMALLOC_MADV_PURGE MADV_FREE
@@ -1954,17 +1949,17 @@ pages_purge(void* addr, size_t length, b
#endif
}
static void*
chunk_recycle(size_t aSize, size_t aAlignment, bool* aZeroed)
{
extent_node_t key;
- size_t alloc_size = aSize + aAlignment - chunksize;
+ size_t alloc_size = aSize + aAlignment - kChunkSize;
// Beware size_t wrap-around.
if (alloc_size < aSize) {
return nullptr;
}
key.addr = nullptr;
key.size = alloc_size;
chunks_mtx.Lock();
extent_node_t* node = gChunksBySize.SearchOrNext(&key);
@@ -2034,36 +2029,36 @@ chunk_recycle(size_t aSize, size_t aAlig
return ret;
}
#ifdef XP_WIN
// On Windows, calls to VirtualAlloc and VirtualFree must be matched, making it
// awkward to recycle allocations of varying sizes. Therefore we only allow
// recycling when the size equals the chunksize, unless deallocation is entirely
// disabled.
-#define CAN_RECYCLE(size) (size == chunksize)
+#define CAN_RECYCLE(size) (size == kChunkSize)
#else
#define CAN_RECYCLE(size) true
#endif
// Allocates `size` bytes of system memory aligned for `alignment`.
// `base` indicates whether the memory will be used for the base allocator
// (e.g. base_alloc).
// `zeroed` is an outvalue that returns whether the allocated memory is
// guaranteed to be full of zeroes. It can be omitted when the caller doesn't
// care about the result.
static void*
chunk_alloc(size_t aSize, size_t aAlignment, bool aBase, bool* aZeroed)
{
void* ret = nullptr;
MOZ_ASSERT(aSize != 0);
- MOZ_ASSERT((aSize & chunksize_mask) == 0);
+ MOZ_ASSERT((aSize & kChunkSizeMask) == 0);
MOZ_ASSERT(aAlignment != 0);
- MOZ_ASSERT((aAlignment & chunksize_mask) == 0);
+ MOZ_ASSERT((aAlignment & kChunkSizeMask) == 0);
// Base allocations can't be fulfilled by recycling because of
// possible deadlock or infinite recursion.
if (CAN_RECYCLE(aSize) && !aBase) {
ret = chunk_recycle(aSize, aAlignment, aZeroed);
}
if (!ret) {
ret = chunk_alloc_mmap(aSize, aAlignment);
@@ -2177,17 +2172,17 @@ chunk_record(void* aChunk, size_t aSize,
}
static void
chunk_dealloc(void* aChunk, size_t aSize, ChunkType aType)
{
MOZ_ASSERT(aChunk);
MOZ_ASSERT(GetChunkOffsetForPtr(aChunk) == 0);
MOZ_ASSERT(aSize != 0);
- MOZ_ASSERT((aSize & chunksize_mask) == 0);
+ MOZ_ASSERT((aSize & kChunkSizeMask) == 0);
gChunkRTree.Unset(aChunk);
if (CAN_RECYCLE(aSize)) {
size_t recycled_so_far = gRecycledSize;
// In case some race condition put us above the limit.
if (recycled_so_far < gRecycleLimit) {
size_t recycle_remaining = gRecycleLimit - recycled_so_far;
@@ -2526,17 +2521,17 @@ arena_t::InitChunk(arena_chunk_t* aChunk
// all it can contain is an arena chunk header (which we're overwriting),
// and zeroed or poisoned memory (because a recycled arena chunk will
// have been emptied before being recycled). In that case, we can get
// away with reusing the chunk as-is, marking all runs as madvised.
size_t flags =
aZeroed ? CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED : CHUNK_MAP_MADVISED;
- mStats.mapped += chunksize;
+ mStats.mapped += kChunkSize;
aChunk->arena = this;
// Claim that no pages are in use, since the header is merely overhead.
aChunk->ndirty = 0;
// Initialize the map to contain one maximal free untouched run.
#ifdef MALLOC_DECOMMIT
@@ -2580,18 +2575,18 @@ arena_t::DeallocChunk(arena_chunk_t* aCh
}
#ifdef MALLOC_DOUBLE_PURGE
if (mChunksMAdvised.ElementProbablyInList(mSpare)) {
mChunksMAdvised.remove(mSpare);
}
#endif
- chunk_dealloc((void*)mSpare, chunksize, ARENA_CHUNK);
- mStats.mapped -= chunksize;
+ chunk_dealloc((void*)mSpare, kChunkSize, ARENA_CHUNK);
+ mStats.mapped -= kChunkSize;
mStats.committed -= arena_chunk_header_npages;
}
// Remove run from the tree of available runs, so that the arena does not use it.
// Dirty page flushing only uses the tree of dirty chunks, so leaving this
// chunk in the chunks_* trees is sufficient for that purpose.
mRunsAvail.Remove(&aChunk->map[arena_chunk_header_npages]);
@@ -2625,17 +2620,17 @@ arena_t::AllocRun(arena_bin_t* aBin, siz
(arena_chunk_header_npages << pagesize_2pow));
// Insert the run into the tree of available runs.
mRunsAvail.Insert(&chunk->map[arena_chunk_header_npages]);
} else {
// No usable runs. Create a new chunk from which to allocate
// the run.
bool zeroed;
arena_chunk_t* chunk =
- (arena_chunk_t*)chunk_alloc(chunksize, chunksize, false, &zeroed);
+ (arena_chunk_t*)chunk_alloc(kChunkSize, kChunkSize, false, &zeroed);
if (!chunk) {
return nullptr;
}
InitChunk(chunk, zeroed);
run = (arena_run_t*)(uintptr_t(chunk) +
(arena_chunk_header_npages << pagesize_2pow));
}
@@ -3261,17 +3256,17 @@ ipalloc(size_t aAlignment, size_t aSize,
// that the bogus run_size value never gets used for
// anything important.
run_size = (aAlignment << 1) - pagesize;
}
if (run_size <= gMaxLargeClass) {
aArena = aArena ? aArena : choose_arena(aSize);
ret = aArena->Palloc(aAlignment, ceil_size, run_size);
- } else if (aAlignment <= chunksize) {
+ } else if (aAlignment <= kChunkSize) {
ret = huge_malloc(ceil_size, false);
} else {
ret = huge_palloc(ceil_size, aAlignment, false);
}
}
MOZ_ASSERT((uintptr_t(ret) & (aAlignment - 1)) == 0);
return ret;
@@ -3873,17 +3868,17 @@ ArenaCollection::CreateArena(bool aIsPri
// End arena.
// ***************************************************************************
// Begin general internal functions.
static void*
huge_malloc(size_t size, bool zero)
{
- return huge_palloc(size, chunksize, zero);
+ return huge_palloc(size, kChunkSize, zero);
}
static void*
huge_palloc(size_t aSize, size_t aAlignment, bool aZero)
{
void* ret;
size_t csize;
size_t psize;
@@ -4514,17 +4509,17 @@ MozJemalloc::jemalloc_stats(jemalloc_sta
}
// Gather runtime settings.
aStats->opt_junk = opt_junk;
aStats->opt_zero = opt_zero;
aStats->quantum = kQuantum;
aStats->small_max = kMaxQuantumClass;
aStats->large_max = gMaxLargeClass;
- aStats->chunksize = chunksize;
+ aStats->chunksize = kChunkSize;
aStats->page_size = pagesize;
aStats->dirty_max = opt_dirty_max;
// Gather current memory usage statistics.
aStats->narenas = 0;
aStats->mapped = 0;
aStats->allocated = 0;
aStats->waste = 0;