Bug 1403843 - Make quantum, small and chunk sizes always static. r?njn
Bug 571209 made many different kinds of sizes static at build time, as
opposed to configurable at run-time. While the dynamic sizes can be
useful to quickly test tweaks to e.g. quantum sizes, a
replace-malloc-built allocator could just as well do the same. This
need, however, is very rare, and doesn't justify keeping the sizes
dynamic on platforms where static sizes can't be used for the page size
because page size may vary depending on kernel options.
So we make every size that doesn't depend on the page size static,
whether MALLOC_STATIC_SIZES is enabled or not.
This makes no practical difference on tier-1 platforms, except Android
aarch64, which will benefit from more static sizes.
--- a/memory/build/mozjemalloc.cpp
+++ b/memory/build/mozjemalloc.cpp
@@ -378,25 +378,43 @@ void *_mmap(void *addr, size_t length, i
* (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP))
*/
#define RUN_BFP 12
/* \/ Implicit binary fixed point. */
#define RUN_MAX_OVRHD 0x0000003dU
#define RUN_MAX_OVRHD_RELAX 0x00001800U
/*
- * When MALLOC_STATIC_SIZES is defined most of the parameters
- * controlling the malloc behavior are defined as compile-time constants
- * for best performance and cannot be altered at runtime.
+ * When MALLOC_STATIC_PAGESIZE is defined, the page size is fixed at
+ * compile-time for better performance, as opposed to determined at
+ * runtime. Some platforms can have different page sizes at runtime
+ * depending on kernel configuration, so they are opted out by default.
*/
#if !defined(__ia64__) && !defined(__sparc__) && !defined(__mips__) && !defined(__aarch64__)
-#define MALLOC_STATIC_SIZES 1
+#define MALLOC_STATIC_PAGESIZE 1
#endif
-#ifdef MALLOC_STATIC_SIZES
+/* Various quantum-related settings. */
+
+#define QUANTUM_DEFAULT (size_t(1) << QUANTUM_2POW_MIN)
+static const size_t quantum = QUANTUM_DEFAULT;
+static const size_t quantum_mask = QUANTUM_DEFAULT - 1;
+
+/* Various bin-related settings. */
+
+static const size_t small_min = (QUANTUM_DEFAULT >> 1) + 1;
+static const size_t small_max = size_t(SMALL_MAX_DEFAULT);
+
+/* Number of (2^n)-spaced tiny bins. */
+static const unsigned ntbins = unsigned(QUANTUM_2POW_MIN - TINY_MIN_2POW);
+
+ /* Number of quantum-spaced bins. */
+static const unsigned nqbins = unsigned(SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN);
+
+#ifdef MALLOC_STATIC_PAGESIZE
/*
* VM page size. It must divide the runtime CPU page size or the code
* will abort.
* Platform specific page size conditions copied from js/public/HeapAPI.h
*/
#if (defined(SOLARIS) || defined(__FreeBSD__)) && \
(defined(__sparc) || defined(__sparcv9) || defined(__ia64))
@@ -404,58 +422,32 @@ void *_mmap(void *addr, size_t length, i
#elif defined(__powerpc64__)
#define pagesize_2pow (size_t(16))
#else
#define pagesize_2pow (size_t(12))
#endif
#define pagesize (size_t(1) << pagesize_2pow)
#define pagesize_mask (pagesize - 1)
-/* Various quantum-related settings. */
-
-#define QUANTUM_DEFAULT (size_t(1) << QUANTUM_2POW_MIN)
-static const size_t quantum = QUANTUM_DEFAULT;
-static const size_t quantum_mask = QUANTUM_DEFAULT - 1;
-
-/* Various bin-related settings. */
-
-static const size_t small_min = (QUANTUM_DEFAULT >> 1) + 1;
-static const size_t small_max = size_t(SMALL_MAX_DEFAULT);
-
/* Max size class for bins. */
static const size_t bin_maxclass = pagesize >> 1;
- /* Number of (2^n)-spaced tiny bins. */
-static const unsigned ntbins = unsigned(QUANTUM_2POW_MIN - TINY_MIN_2POW);
-
- /* Number of quantum-spaced bins. */
-static const unsigned nqbins = unsigned(SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN);
-
/* Number of (2^n)-spaced sub-page bins. */
static const unsigned nsbins = unsigned(pagesize_2pow - SMALL_MAX_2POW_DEFAULT - 1);
-#else /* !MALLOC_STATIC_SIZES */
+#else /* !MALLOC_STATIC_PAGESIZE */
/* VM page size. */
static size_t pagesize;
static size_t pagesize_mask;
static size_t pagesize_2pow;
/* Various bin-related settings. */
static size_t bin_maxclass; /* Max size class for bins. */
-static unsigned ntbins; /* Number of (2^n)-spaced tiny bins. */
-static unsigned nqbins; /* Number of quantum-spaced bins. */
static unsigned nsbins; /* Number of (2^n)-spaced sub-page bins. */
-static size_t small_min;
-static size_t small_max;
-
-/* Various quantum-related settings. */
-static size_t quantum;
-static size_t quantum_mask; /* (quantum - 1). */
-
#endif
/* Various chunk-related settings. */
/*
* Compute the header size such that it is large enough to contain the page map
* and enough nodes for the worst case: one node per non-header page plus one
* extra for situations where we briefly have one more node allocated than we
@@ -473,17 +465,17 @@ static size_t quantum_mask; /* (quantum
(chunksize - (arena_chunk_header_npages << pagesize_2pow))
/*
* Recycle at most 128 chunks. With 1 MiB chunks, this means we retain at most
* 6.25% of the process address space on a 32-bit OS for later use.
*/
#define CHUNK_RECYCLE_LIMIT 128
-#ifdef MALLOC_STATIC_SIZES
+#ifdef MALLOC_STATIC_PAGESIZE
#define CHUNKSIZE_DEFAULT ((size_t) 1 << CHUNK_2POW_DEFAULT)
static const size_t chunksize = CHUNKSIZE_DEFAULT;
static const size_t chunksize_mask = CHUNKSIZE_DEFAULT - 1;
static const size_t chunk_npages = CHUNKSIZE_DEFAULT >> pagesize_2pow;
#define arena_chunk_header_npages calculate_arena_header_pages()
#define arena_maxclass calculate_arena_maxclass()
static const size_t recycle_limit = CHUNK_RECYCLE_LIMIT * CHUNKSIZE_DEFAULT;
#else
@@ -1128,25 +1120,16 @@ const uint8_t kAllocPoison = 0xe5;
static bool opt_junk = true;
static bool opt_zero = false;
#else
static const bool opt_junk = false;
static const bool opt_zero = false;
#endif
static size_t opt_dirty_max = DIRTY_MAX_DEFAULT;
-#ifdef MALLOC_STATIC_SIZES
-#define opt_quantum_2pow QUANTUM_2POW_MIN
-#define opt_small_max_2pow SMALL_MAX_2POW_DEFAULT
-#define opt_chunk_2pow CHUNK_2POW_DEFAULT
-#else
-static size_t opt_quantum_2pow = QUANTUM_2POW_MIN;
-static size_t opt_small_max_2pow = SMALL_MAX_2POW_DEFAULT;
-static size_t opt_chunk_2pow = CHUNK_2POW_DEFAULT;
-#endif
/******************************************************************************/
/*
* Begin forward declarations.
*/
static void *chunk_alloc(size_t size, size_t alignment, bool base, bool *zeroed=nullptr);
static void chunk_dealloc(void *chunk, size_t size, ChunkType chunk_type);
@@ -3208,22 +3191,22 @@ arena_t::MallocSmall(size_t aSize, bool
* stats accuracy.
*/
if (aSize < (1U << TINY_MIN_2POW)) {
aSize = 1U << TINY_MIN_2POW;
}
} else if (aSize <= small_max) {
/* Quantum-spaced. */
aSize = QUANTUM_CEILING(aSize);
- bin = &mBins[ntbins + (aSize >> opt_quantum_2pow) - 1];
+ bin = &mBins[ntbins + (aSize >> QUANTUM_2POW_MIN) - 1];
} else {
/* Sub-page. */
aSize = pow2_ceil(aSize);
bin = &mBins[ntbins + nqbins
- + (ffs((int)(aSize >> opt_small_max_2pow)) - 2)];
+ + (ffs((int)(aSize >> SMALL_MAX_2POW_DEFAULT)) - 2)];
}
MOZ_DIAGNOSTIC_ASSERT(aSize == bin->reg_size);
malloc_spin_lock(&mLock);
if ((run = bin->runcur) && run->nfree > 0) {
ret = MallocBinEasy(bin, run);
} else {
ret = MallocBinHard(bin);
@@ -3914,18 +3897,18 @@ arena_ralloc(void* aPtr, size_t aSize, s
if (aSize < small_min) {
if (aOldSize < small_min &&
ffs((int)(pow2_ceil(aSize) >> (TINY_MIN_2POW + 1))) ==
ffs((int)(pow2_ceil(aOldSize) >> (TINY_MIN_2POW + 1)))) {
goto IN_PLACE; /* Same size class. */
}
} else if (aSize <= small_max) {
if (aOldSize >= small_min && aOldSize <= small_max &&
- (QUANTUM_CEILING(aSize) >> opt_quantum_2pow) ==
- (QUANTUM_CEILING(aOldSize) >> opt_quantum_2pow)) {
+ (QUANTUM_CEILING(aSize) >> QUANTUM_2POW_MIN) ==
+ (QUANTUM_CEILING(aOldSize) >> QUANTUM_2POW_MIN)) {
goto IN_PLACE; /* Same size class. */
}
} else if (aSize <= bin_maxclass) {
if (aOldSize > small_max && aOldSize <= bin_maxclass &&
pow2_ceil(aSize) == pow2_ceil(aOldSize)) {
goto IN_PLACE; /* Same size class. */
}
} else if (aOldSize > bin_maxclass && aOldSize <= arena_maxclass) {
@@ -4373,17 +4356,17 @@ malloc_init_hard(void)
if (!thread_arena.init()) {
return false;
}
/* Get page size and number of CPUs */
result = GetKernelPageSize();
/* We assume that the page size is a power of 2. */
MOZ_ASSERT(((result - 1) & result) == 0);
-#ifdef MALLOC_STATIC_SIZES
+#ifdef MALLOC_STATIC_PAGESIZE
if (pagesize % (size_t) result) {
_malloc_message(_getprogname(),
"Compile-time page size does not divide the runtime one.\n");
MOZ_CRASH();
}
#else
pagesize = (size_t) result;
pagesize_mask = (size_t) result - 1;
@@ -4427,53 +4410,16 @@ MALLOC_OUT:
#ifdef MOZ_DEBUG
case 'j':
opt_junk = false;
break;
case 'J':
opt_junk = true;
break;
#endif
-#ifndef MALLOC_STATIC_SIZES
- case 'k':
- /*
- * Chunks always require at least one
- * header page, so chunks can never be
- * smaller than two pages.
- */
- if (opt_chunk_2pow > pagesize_2pow + 1)
- opt_chunk_2pow--;
- break;
- case 'K':
- if (opt_chunk_2pow + 1 <
- (sizeof(size_t) << 3))
- opt_chunk_2pow++;
- break;
-#endif
-#ifndef MALLOC_STATIC_SIZES
- case 'q':
- if (opt_quantum_2pow > QUANTUM_2POW_MIN)
- opt_quantum_2pow--;
- break;
- case 'Q':
- if (opt_quantum_2pow < pagesize_2pow -
- 1)
- opt_quantum_2pow++;
- break;
- case 's':
- if (opt_small_max_2pow >
- QUANTUM_2POW_MIN)
- opt_small_max_2pow--;
- break;
- case 'S':
- if (opt_small_max_2pow < pagesize_2pow
- - 1)
- opt_small_max_2pow++;
- break;
-#endif
#ifdef MOZ_DEBUG
case 'z':
opt_zero = false;
break;
case 'Z':
opt_zero = true;
break;
#endif
@@ -4487,43 +4433,23 @@ MALLOC_OUT:
"in malloc options: '", cbuf,
"'\n");
}
}
}
}
}
-#ifndef MALLOC_STATIC_SIZES
- /* Set variables according to the value of opt_small_max_2pow. */
- if (opt_small_max_2pow < opt_quantum_2pow) {
- opt_small_max_2pow = opt_quantum_2pow;
- }
- small_max = (1U << opt_small_max_2pow);
-
+#ifndef MALLOC_STATIC_PAGESIZE
/* Set bin-related variables. */
bin_maxclass = (pagesize >> 1);
- MOZ_ASSERT(opt_quantum_2pow >= TINY_MIN_2POW);
- ntbins = opt_quantum_2pow - TINY_MIN_2POW;
- MOZ_ASSERT(ntbins <= opt_quantum_2pow);
- nqbins = (small_max >> opt_quantum_2pow);
- nsbins = pagesize_2pow - opt_small_max_2pow - 1;
-
- /* Set variables according to the value of opt_quantum_2pow. */
- quantum = (1U << opt_quantum_2pow);
- quantum_mask = quantum - 1;
- if (ntbins > 0) {
- small_min = (quantum >> 1) + 1;
- } else {
- small_min = 1;
- }
- MOZ_ASSERT(small_min <= quantum);
-
- /* Set variables according to the value of opt_chunk_2pow. */
- chunksize = (1LU << opt_chunk_2pow);
+ nsbins = pagesize_2pow - SMALL_MAX_2POW_DEFAULT - 1;
+
+ /* Set variables according to the value of CHUNK_2POW_DEFAULT. */
+ chunksize = (1LU << CHUNK_2POW_DEFAULT);
chunksize_mask = chunksize - 1;
chunk_npages = (chunksize >> pagesize_2pow);
arena_chunk_header_npages = calculate_arena_header_pages();
arena_maxclass = calculate_arena_maxclass();
recycle_limit = CHUNK_RECYCLE_LIMIT * chunksize;
#endif
@@ -4573,17 +4499,17 @@ MALLOC_OUT:
* reset to the default value for the main arena. */
gMainArena->mMaxDirty = opt_dirty_max;
/*
* Assign the initial arena to the initial thread.
*/
thread_arena.set(gMainArena);
- chunk_rtree = malloc_rtree_new((SIZEOF_PTR << 3) - opt_chunk_2pow);
+ chunk_rtree = malloc_rtree_new((SIZEOF_PTR << 3) - CHUNK_2POW_DEFAULT);
if (!chunk_rtree) {
return true;
}
malloc_initialized = true;
#if !defined(XP_WIN) && !defined(XP_DARWIN)
/* Prevent potential deadlock on malloc locks after fork. */