--- a/js/src/wasm/WasmBaselineCompile.cpp
+++ b/js/src/wasm/WasmBaselineCompile.cpp
@@ -140,25 +140,16 @@ typedef unsigned BitSize;
//
// When those registers are volatile, the baseline compiler will reload them
// after the call (it will restore the Tls register from the save slot and load
// the other two from the Tls data).
enum class UseABI { Wasm, System };
enum class InterModule { False = false, True = true };
-#ifdef JS_CODEGEN_ARM64
-// FIXME: This is not correct, indeed for ARM64 there is no reliable
-// StackPointer and we'll need to change the abstractions that use
-// SP-relative addressing. There's a guard in emitFunction() below to
-// prevent this workaround from having any consequence. This hack
-// exists only as a stopgap; there is no ARM64 JIT support yet.
-static const Register StackPointer = RealStackPointer;
-#endif
-
#ifdef JS_CODEGEN_X86
// The selection of EBX here steps gingerly around: the need for EDX
// to be allocatable for multiply/divide; ECX to be allocatable for
// shift/rotate; EAX (= ReturnReg) to be allocatable as the joinreg;
// EBX not being one of the WasmTableCall registers; and needing a
// temp register for load/store that has a single-byte persona.
//
// The compiler assumes that ScratchRegX86 has a single-byte persona.
@@ -193,119 +184,16 @@ struct RegTypeOf {
template<> struct RegTypeOf<MIRType::Float32> {
static constexpr RegTypeName value = RegTypeName::Float32;
};
template<> struct RegTypeOf<MIRType::Double> {
static constexpr RegTypeName value = RegTypeName::Float64;
};
-BaseLocalIter::BaseLocalIter(const ValTypeVector& locals,
- size_t argsLength,
- bool debugEnabled)
- : locals_(locals),
- argsLength_(argsLength),
- argsRange_(locals.begin(), argsLength),
- argsIter_(argsRange_),
- index_(0),
- localSize_(debugEnabled ? DebugFrame::offsetOfFrame() : 0),
- reservedSize_(localSize_),
- done_(false)
-{
- MOZ_ASSERT(argsLength <= locals.length());
-
- settle();
-}
-
-int32_t
-BaseLocalIter::pushLocal(size_t nbytes)
-{
- if (nbytes == 8)
- localSize_ = AlignBytes(localSize_, 8u);
- else if (nbytes == 16)
- localSize_ = AlignBytes(localSize_, 16u);
- localSize_ += nbytes;
- return localSize_; // Locals grow down so capture base address
-}
-
-void
-BaseLocalIter::settle()
-{
- if (index_ < argsLength_) {
- MOZ_ASSERT(!argsIter_.done());
- mirType_ = argsIter_.mirType();
- switch (mirType_) {
- case MIRType::Int32:
- if (argsIter_->argInRegister())
- frameOffset_ = pushLocal(4);
- else
- frameOffset_ = -(argsIter_->offsetFromArgBase() + sizeof(Frame));
- break;
- case MIRType::Int64:
- if (argsIter_->argInRegister())
- frameOffset_ = pushLocal(8);
- else
- frameOffset_ = -(argsIter_->offsetFromArgBase() + sizeof(Frame));
- break;
- case MIRType::Double:
- if (argsIter_->argInRegister())
- frameOffset_ = pushLocal(8);
- else
- frameOffset_ = -(argsIter_->offsetFromArgBase() + sizeof(Frame));
- break;
- case MIRType::Float32:
- if (argsIter_->argInRegister())
- frameOffset_ = pushLocal(4);
- else
- frameOffset_ = -(argsIter_->offsetFromArgBase() + sizeof(Frame));
- break;
- default:
- MOZ_CRASH("Argument type");
- }
- return;
- }
-
- MOZ_ASSERT(argsIter_.done());
- if (index_ < locals_.length()) {
- switch (locals_[index_]) {
- case ValType::I32:
- mirType_ = jit::MIRType::Int32;
- frameOffset_ = pushLocal(4);
- break;
- case ValType::F32:
- mirType_ = jit::MIRType::Float32;
- frameOffset_ = pushLocal(4);
- break;
- case ValType::F64:
- mirType_ = jit::MIRType::Double;
- frameOffset_ = pushLocal(8);
- break;
- case ValType::I64:
- mirType_ = jit::MIRType::Int64;
- frameOffset_ = pushLocal(8);
- break;
- default:
- MOZ_CRASH("Compiler bug: Unexpected local type");
- }
- return;
- }
-
- done_ = true;
-}
-
-void
-BaseLocalIter::operator++(int)
-{
- MOZ_ASSERT(!done_);
- index_++;
- if (!argsIter_.done())
- argsIter_++;
- settle();
-}
-
// The strongly typed register wrappers are especially useful to distinguish
// float registers from double registers, but they also clearly distinguish
// 32-bit registers from 64-bit register pairs on 32-bit systems.
struct RegI32 : public Register
{
RegI32() : Register(Register::Invalid()) {}
explicit RegI32(Register reg) : Register(reg) {}
@@ -781,17 +669,18 @@ class ScratchF32 : public ScratchFloat32
#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
// On x86 we do not have a dedicated masm scratch register; on ARM, we need one
// in addition to the one defined by masm because masm uses it too often.
class ScratchI32
{
# ifdef DEBUG
BaseRegAlloc& ra;
public:
- explicit ScratchI32(BaseRegAlloc& ra) : ra(ra) {
+ explicit ScratchI32(BaseRegAlloc& ra) : ra(ra)
+ {
MOZ_ASSERT(!ra.scratchRegisterTaken());
ra.setScratchRegisterTaken(true);
}
~ScratchI32() {
MOZ_ASSERT(ra.scratchRegisterTaken());
ra.setScratchRegisterTaken(false);
}
# else
@@ -821,62 +710,652 @@ class ScratchI32 : public ScratchRegiste
// no other register will do. And we would normally have to allocate that
// register using ScratchI32 since normally the scratch register is EBX.
// But the whole point of ScratchI32 is to hide that relationship. By using
// the ScratchEBX alias, we document that at that point we require the
// scratch register to be EBX.
typedef ScratchI32 ScratchEBX;
#endif
-class BaseCompiler final : public BaseCompilerInterface
-{
- typedef Vector<NonAssertingLabel, 8, SystemAllocPolicy> LabelVector;
- typedef Vector<MIRType, 8, SystemAllocPolicy> MIRTypeVector;
+// The stack frame.
+//
+// The frame has four parts: the Header, comprising the Frame and DebugFrame
+// elements; the Local area, allocated below the header with various forms of
+// alignment; the Stack area, comprising the temporary storage the compiler uses
+// for register spilling, allocated below the Local area; and the Arguments
+// area, comprising memory allocated for outgoing calls, allocated below the
+// stack area.
+//
+// The header is addressed off the stack pointer. masm.framePushed() is always
+// correct, and masm.getStackPointer() + masm.framePushed() always addresses the
+// Frame, with the DebugFrame optionally below it.
+//
+// The local area is laid out by BaseLocalIter and is allocated and deallocated
+// by standard prologue and epilogue functions that manipulate the stack
+// pointer, but it is accessed via BaseStackFrame.
+//
+// The stack area is maintained by and accessed via BaseStackFrame. On some
+// systems, the stack memory may be allocated in chunks because the SP needs a
+// specific alignment.
+//
+// The arguments area is allocated and deallocated via BaseStackFrame (see
+// comments later) but is accessed directly off the stack pointer.
+
+// BaseLocalIter iterates over a vector of types of locals and provides offsets
+// from the Frame address for those locals, and associated data.
+//
+// The implementation of BaseLocalIter is the property of the BaseStackFrame.
+// But it is also exposed for eg the debugger to use.
+
+BaseLocalIter::BaseLocalIter(const ValTypeVector& locals,
+ size_t argsLength,
+ bool debugEnabled)
+ : locals_(locals),
+ argsLength_(argsLength),
+ argsRange_(locals.begin(), argsLength),
+ argsIter_(argsRange_),
+ index_(0),
+ localSize_(debugEnabled ? DebugFrame::offsetOfFrame() : 0),
+ reservedSize_(localSize_),
+ done_(false)
+{
+ MOZ_ASSERT(argsLength <= locals.length());
+
+ settle();
+}
+
+int32_t
+BaseLocalIter::pushLocal(size_t nbytes)
+{
+ if (nbytes == 8)
+ localSize_ = AlignBytes(localSize_, 8u);
+ else if (nbytes == 16)
+ localSize_ = AlignBytes(localSize_, 16u);
+ localSize_ += nbytes;
+ return localSize_; // Locals grow down so capture base address
+}
+
+void
+BaseLocalIter::settle()
+{
+ if (index_ < argsLength_) {
+ MOZ_ASSERT(!argsIter_.done());
+ mirType_ = argsIter_.mirType();
+ switch (mirType_) {
+ case MIRType::Int32:
+ if (argsIter_->argInRegister())
+ frameOffset_ = pushLocal(4);
+ else
+ frameOffset_ = -(argsIter_->offsetFromArgBase() + sizeof(Frame));
+ break;
+ case MIRType::Int64:
+ if (argsIter_->argInRegister())
+ frameOffset_ = pushLocal(8);
+ else
+ frameOffset_ = -(argsIter_->offsetFromArgBase() + sizeof(Frame));
+ break;
+ case MIRType::Double:
+ if (argsIter_->argInRegister())
+ frameOffset_ = pushLocal(8);
+ else
+ frameOffset_ = -(argsIter_->offsetFromArgBase() + sizeof(Frame));
+ break;
+ case MIRType::Float32:
+ if (argsIter_->argInRegister())
+ frameOffset_ = pushLocal(4);
+ else
+ frameOffset_ = -(argsIter_->offsetFromArgBase() + sizeof(Frame));
+ break;
+ default:
+ MOZ_CRASH("Argument type");
+ }
+ return;
+ }
+
+ MOZ_ASSERT(argsIter_.done());
+ if (index_ < locals_.length()) {
+ switch (locals_[index_]) {
+ case ValType::I32:
+ mirType_ = jit::MIRType::Int32;
+ frameOffset_ = pushLocal(4);
+ break;
+ case ValType::F32:
+ mirType_ = jit::MIRType::Float32;
+ frameOffset_ = pushLocal(4);
+ break;
+ case ValType::F64:
+ mirType_ = jit::MIRType::Double;
+ frameOffset_ = pushLocal(8);
+ break;
+ case ValType::I64:
+ mirType_ = jit::MIRType::Int64;
+ frameOffset_ = pushLocal(8);
+ break;
+ default:
+ MOZ_CRASH("Compiler bug: Unexpected local type");
+ }
+ return;
+ }
+
+ done_ = true;
+}
+
+void
+BaseLocalIter::operator++(int)
+{
+ MOZ_ASSERT(!done_);
+ index_++;
+ if (!argsIter_.done())
+ argsIter_++;
+ settle();
+}
+
+// Abstraction of the baseline compiler's stack frame (except for the Frame /
+// DebugFrame parts). See comments above for more.
+
+class BaseStackFrame
+{
+ MacroAssembler& masm;
+
+ // Size of local area in bytes (stable after beginFunction).
+ uint32_t localSize_;
+
+ // Low byte offset of local area for true locals (not parameters).
+ uint32_t varLow_;
+
+ // High byte offset + 1 of local area for true locals.
+ uint32_t varHigh_;
+
+ // The largest stack height, not necessarily zero-based. Read this for its
+ // true value only when code generation is finished.
+ uint32_t maxStackHeight_;
+
+ // Patch point where we check for stack overflow.
+ CodeOffset stackAddOffset_;
+
+ // The stack pointer, cached for brevity.
+ Register sp_;
+
+ public:
+
+ explicit BaseStackFrame(MacroAssembler& masm)
+ : masm(masm),
+ localSize_(UINT32_MAX),
+ varLow_(UINT32_MAX),
+ varHigh_(UINT32_MAX),
+ maxStackHeight_(0),
+ stackAddOffset_(0),
+ sp_(masm.getStackPointer())
+ {}
+
+ //////////////////////////////////////////////////////////////////////
+ //
+ // The local area.
+
+ // Locals - the static part of the frame.
struct Local
{
- Local() : type_(MIRType::None), offs_(UINT32_MAX) {}
- Local(MIRType type, uint32_t offs) : type_(type), offs_(offs) {}
-
- void init(MIRType type_, uint32_t offs_) {
- this->type_ = type_;
- this->offs_ = offs_;
+ // Type of the value.
+ const MIRType type;
+
+ // Byte offset from Frame "into" the locals, ie positive for true locals
+ // and negative for incoming args that read directly from the arg area.
+ // It assumes the stack is growing down and that locals are on the stack
+ // at lower addresses than Frame, and is the offset from Frame of the
+ // lowest-addressed byte of the local.
+ const int32_t offs;
+
+ Local(MIRType type, int32_t offs) : type(type), offs(offs) {}
+ };
+
+ typedef Vector<Local, 8, SystemAllocPolicy> LocalVector;
+
+ private:
+
+ // Offset off of sp_ for `local`.
+ int32_t localOffset(Local& local) {
+ return localOffset(local.offs);
+ }
+
+ // Offset off of sp_ for a local with offset `offset` from Frame.
+ int32_t localOffset(int32_t offset) {
+ return masm.framePushed() - offset;
+ }
+
+ public:
+
+ void endFunctionPrologue() {
+ MOZ_ASSERT(masm.framePushed() == localSize_);
+ MOZ_ASSERT(localSize_ != UINT32_MAX);
+ MOZ_ASSERT(localSize_ % WasmStackAlignment == 0);
+ maxStackHeight_ = localSize_;
+ }
+
+ // Initialize `localInfo` based on the types of `locals` and `args`.
+ bool setupLocals(LocalVector& localInfo, const ValTypeVector& locals, const ValTypeVector& args,
+ bool debugEnabled)
+ {
+ MOZ_ASSERT(maxStackHeight_ != UINT32_MAX);
+
+ if (!localInfo.reserve(locals.length()))
+ return false;
+
+ DebugOnly<uint32_t> index = 0;
+ BaseLocalIter i(locals, args.length(), debugEnabled);
+ varLow_ = i.reservedSize();
+ for (; !i.done() && i.index() < args.length(); i++) {
+ MOZ_ASSERT(i.isArg());
+ MOZ_ASSERT(i.index() == index);
+ localInfo.infallibleEmplaceBack(i.mirType(), i.frameOffset());
+ varLow_ = i.currentLocalSize();
+ index++;
+ }
+
+ varHigh_ = varLow_;
+ for (; !i.done() ; i++) {
+ MOZ_ASSERT(!i.isArg());
+ MOZ_ASSERT(i.index() == index);
+ localInfo.infallibleEmplaceBack(i.mirType(), i.frameOffset());
+ varHigh_ = i.currentLocalSize();
+ index++;
}
- MIRType type_; // Type of the value, or MIRType::None
- uint32_t offs_; // Zero-based frame offset of value, or UINT32_MAX
-
- MIRType type() const { MOZ_ASSERT(type_ != MIRType::None); return type_; }
- uint32_t offs() const { MOZ_ASSERT(offs_ != UINT32_MAX); return offs_; }
- };
+ localSize_ = AlignBytes(varHigh_, WasmStackAlignment);
+
+ return true;
+ }
+
+ // The fixed amount of memory, in bytes, allocated on the stack below the
+ // Frame for purposes such as locals and other fixed values. Includes all
+ // necessary alignment.
+
+ uint32_t initialSize() const {
+ MOZ_ASSERT(localSize_ != UINT32_MAX);
+
+ return localSize_;
+ }
+
+ void zeroLocals(BaseRegAlloc& ra);
+
+ void loadLocalI32(RegI32 r, Local& local) {
+ masm.load32(Address(sp_, localOffset(local)), r);
+ }
+
+#ifndef JS_64BIT
+ void loadLocalI64Low(RegI32 r, Local& local) {
+ masm.load32(Address(sp_, localOffset(local) + INT64LOW_OFFSET), r);
+ }
+
+ void loadLocalI64High(RegI32 r, Local& local) {
+ masm.load32(Address(sp_, localOffset(local) + INT64HIGH_OFFSET), r);
+ }
+#endif
+
+ void loadLocalI64(RegI64 r, Local& local) {
+ masm.load64(Address(sp_, localOffset(local)), r);
+ }
+
+ void loadLocalPtr(Register r, Local& local) {
+ masm.loadPtr(Address(sp_, localOffset(local)), r);
+ }
+
+ void loadLocalF64(RegF64 r, Local& local) {
+ masm.loadDouble(Address(sp_, localOffset(local)), r);
+ }
+
+ void loadLocalF32(RegF32 r, Local& local) {
+ masm.loadFloat32(Address(sp_, localOffset(local)), r);
+ }
+
+ void storeLocalI32(RegI32 r, Local& local) {
+ masm.store32(r, Address(sp_, localOffset(local)));
+ }
+
+ void storeLocalI64(RegI64 r, Local& local) {
+ masm.store64(r, Address(sp_, localOffset(local)));
+ }
+
+ void storeLocalPtr(Register r, Local& local) {
+ masm.storePtr(r, Address(sp_, localOffset(local)));
+ }
+
+ void storeLocalF64(RegF64 r, Local& local) {
+ masm.storeDouble(r, Address(sp_, localOffset(local)));
+ }
+
+ void storeLocalF32(RegF32 r, Local& local) {
+ masm.storeFloat32(r, Address(sp_, localOffset(local)));
+ }
+
+ //////////////////////////////////////////////////////////////////////
+ //
+ // The stack area - the dynamic part of the frame.
+
+ private:
+
+ // Offset off of sp_ for the slot at stack area location `offset`
+ int32_t stackOffset(int32_t offset) {
+ return masm.framePushed() - offset;
+ }
+
+ public:
+
+ // Sizes of items in the stack area.
+ //
+ // The size values come from the implementations of Push() in
+ // MacroAssembler-x86-shared.cpp and MacroAssembler-arm-shared.cpp, and from
+ // VFPRegister::size() in Architecture-arm.h.
+ //
+ // On ARM unlike on x86 we push a single for float.
+
+ static const size_t StackSizeOfPtr = sizeof(intptr_t);
+ static const size_t StackSizeOfInt64 = sizeof(int64_t);
+#ifdef JS_CODEGEN_ARM
+ static const size_t StackSizeOfFloat = sizeof(float);
+#else
+ static const size_t StackSizeOfFloat = sizeof(double);
+#endif
+ static const size_t StackSizeOfDouble = sizeof(double);
+
+ // We won't know until after we've generated code how big the frame will
+ // be (we may need arbitrary spill slots and outgoing param slots) so
+ // emit a patchable add that is patched in endFunction().
+ //
+ // ScratchReg may be used by branchPtr(), so use ABINonArgReg0/1 for
+ // temporaries.
+
+ void allocStack(Label* stackOverflowLabel) {
+ stackAddOffset_ = masm.add32ToPtrWithPatch(masm.getStackPointer(), ABINonArgReg0);
+ masm.wasmEmitStackCheck(ABINonArgReg0, ABINonArgReg1, stackOverflowLabel);
+ }
+
+ void patchAllocStack() {
+ masm.patchAdd32ToPtr(stackAddOffset_,
+ Imm32(-int32_t(maxStackHeight_ - localSize_)));
+ }
+
+ // Very large frames are implausible, probably an attack.
+ bool checkStackHeight() {
+ // 256KB ought to be enough for anyone.
+ return maxStackHeight_ <= 256 * 1024;
+ }
+
+ // The current height of the stack area, not necessarily zero-based.
+ uint32_t stackHeight() const {
+ return masm.framePushed();
+ }
+
+ // Set the frame height. This must only be called with a value returned
+ // from stackHeight().
+ void setStackHeight(uint32_t amount) {
+ masm.setFramePushed(amount);
+ }
+
+ uint32_t pushPtr(Register r) {
+ DebugOnly<uint32_t> stackBefore = stackHeight();
+ masm.Push(r);
+ maxStackHeight_ = Max(maxStackHeight_, stackHeight());
+ MOZ_ASSERT(stackBefore + StackSizeOfPtr == stackHeight());
+ return stackHeight();
+ }
+
+ uint32_t pushFloat(FloatRegister r) {
+ DebugOnly<uint32_t> stackBefore = stackHeight();
+ masm.Push(r);
+ maxStackHeight_ = Max(maxStackHeight_, stackHeight());
+ MOZ_ASSERT(stackBefore + StackSizeOfFloat == stackHeight());
+ return stackHeight();
+ }
+
+ uint32_t pushDouble(FloatRegister r) {
+ DebugOnly<uint32_t> stackBefore = stackHeight();
+ masm.Push(r);
+ maxStackHeight_ = Max(maxStackHeight_, stackHeight());
+ MOZ_ASSERT(stackBefore + StackSizeOfDouble == stackHeight());
+ return stackHeight();
+ }
+
+ void popPtr(Register r) {
+ DebugOnly<uint32_t> stackBefore = stackHeight();
+ masm.Pop(r);
+ MOZ_ASSERT(stackBefore - StackSizeOfPtr == stackHeight());
+ }
+
+ void popFloat(FloatRegister r) {
+ DebugOnly<uint32_t> stackBefore = stackHeight();
+ masm.Pop(r);
+ MOZ_ASSERT(stackBefore - StackSizeOfFloat == stackHeight());
+ }
+
+ void popDouble(FloatRegister r) {
+ DebugOnly<uint32_t> stackBefore = stackHeight();
+ masm.Pop(r);
+ MOZ_ASSERT(stackBefore - StackSizeOfDouble == stackHeight());
+ }
+
+ void popBytes(size_t bytes) {
+ if (bytes > 0)
+ masm.freeStack(bytes);
+ }
+
+ // Before branching to an outer control label, pop the execution
+ // stack to the level expected by that region, but do not free the
+ // stack as that will happen as compilation leaves the block.
+
+ void popStackBeforeBranch(uint32_t destStackHeight) {
+ uint32_t stackHere = stackHeight();
+ if (stackHere > destStackHeight)
+ masm.addToStackPtr(Imm32(stackHere - destStackHeight));
+ }
+
+ bool willPopStackBeforeBranch(uint32_t destStackHeight) {
+ uint32_t stackHere = stackHeight();
+ return stackHere > destStackHeight;
+ }
+
+ // Before exiting a nested control region, pop the execution stack
+ // to the level expected by the nesting region, and free the
+ // stack.
+
+ void popStackOnBlockExit(uint32_t destStackHeight, bool deadCode) {
+ uint32_t stackHere = stackHeight();
+ if (stackHere > destStackHeight) {
+ if (deadCode)
+ masm.setFramePushed(destStackHeight);
+ else
+ masm.freeStack(stackHere - destStackHeight);
+ }
+ }
+
+ void loadStackI32(RegI32 r, int32_t offset) {
+ masm.load32(Address(sp_, stackOffset(offset)), r);
+ }
+
+ void loadStackI64(RegI64 r, int32_t offset) {
+ masm.load64(Address(sp_, stackOffset(offset)), r);
+ }
+
+#ifndef JS_64BIT
+ void loadStackI64Low(RegI32 r, int32_t offset) {
+ masm.load32(Address(sp_, stackOffset(offset - INT64LOW_OFFSET)), r);
+ }
+
+ void loadStackI64High(RegI32 r, int32_t offset) {
+ masm.load32(Address(sp_, stackOffset(offset - INT64HIGH_OFFSET)), r);
+ }
+#endif
+
+ // Disambiguation: this loads a "Ptr" value from the stack, it does not load
+ // the "StackPtr".
+
+ void loadStackPtr(Register r, int32_t offset) {
+ masm.loadPtr(Address(sp_, stackOffset(offset)), r);
+ }
+
+ void loadStackF64(RegF64 r, int32_t offset) {
+ masm.loadDouble(Address(sp_, stackOffset(offset)), r);
+ }
+
+ void loadStackF32(RegF32 r, int32_t offset) {
+ masm.loadFloat32(Address(sp_, stackOffset(offset)), r);
+ }
+
+ //////////////////////////////////////////////////////////////////////
+ //
+ // The argument area - for outgoing calls.
+ //
+ // We abstract these operations as an optimization: we can merge the freeing
+ // of the argument area and dropping values off the stack after a call. But
+ // they always amount to manipulating the real stack pointer by some amount.
+
+ // This is always equivalent to a masm.reserveStack() call.
+ void allocArgArea(size_t size) {
+ if (size)
+ masm.reserveStack(size);
+ }
+
+ // This is always equivalent to a sequence of masm.freeStack() calls.
+ void freeArgAreaAndPopBytes(size_t argSize, size_t dropSize) {
+ if (argSize + dropSize)
+ masm.freeStack(argSize + dropSize);
+ }
+};
+
+void
+BaseStackFrame::zeroLocals(BaseRegAlloc& ra)
+{
+ MOZ_ASSERT(varLow_ != UINT32_MAX);
+
+ if (varLow_ == varHigh_)
+ return;
+
+ static const uint32_t wordSize = sizeof(void*);
+
+ // The adjustments to 'low' by the size of the item being stored compensates
+ // for the fact that locals offsets are the offsets from Frame to the bytes
+ // directly "above" the locals in the locals area. See comment at Local.
+
+ // On 64-bit systems we may have 32-bit alignment for the local area as it
+ // may be preceded by parameters and prologue/debug data.
+
+ uint32_t low = varLow_;
+ if (low % wordSize) {
+ masm.store32(Imm32(0), Address(sp_, localOffset(low + 4)));
+ low += 4;
+ }
+ MOZ_ASSERT(low % wordSize == 0);
+
+ const uint32_t high = AlignBytes(varHigh_, wordSize);
+ MOZ_ASSERT(high <= localSize_, "localSize_ should be aligned at least that");
+
+ // An unrollLimit of 16 is chosen so that we only need an 8-bit signed
+ // immediate to represent the offset in the store instructions in the loop
+ // on x64.
+
+ const uint32_t unrollLimit = 16;
+ const uint32_t initWords = (high - low) / wordSize;
+ const uint32_t tailWords = initWords % unrollLimit;
+ const uint32_t loopHigh = high - (tailWords * wordSize);
+
+ // With only one word to initialize, just store an immediate zero.
+
+ if (initWords == 1) {
+ masm.storePtr(ImmWord(0), Address(sp_, localOffset(low + wordSize)));
+ return;
+ }
+
+ // For other cases, it's best to have a zero in a register.
+ //
+ // One can do more here with SIMD registers (store 16 bytes at a time) or
+ // with instructions like STRD on ARM (store 8 bytes at a time), but that's
+ // for another day.
+
+ RegI32 zero = ra.needI32();
+ masm.mov(ImmWord(0), zero);
+
+ // For the general case we want to have a loop body of unrollLimit stores
+ // and then a tail of less than unrollLimit stores. When initWords is less
+ // than 2*unrollLimit the loop trip count is at most 1 and there is no
+ // benefit to having the pointer calculations and the compare-and-branch.
+ // So we completely unroll when we have initWords < 2 * unrollLimit. (In
+ // this case we'll end up using 32-bit offsets on x64 for up to half of the
+ // stores, though.)
+
+ // Fully-unrolled case.
+
+ if (initWords < 2 * unrollLimit) {
+ for (uint32_t i = low; i < high; i += wordSize)
+ masm.storePtr(zero, Address(sp_, localOffset(i + wordSize)));
+ ra.freeI32(zero);
+ return;
+ }
+
+ // Unrolled loop with a tail. Stores will use negative offsets. That's OK
+ // for x86 and ARM, at least.
+
+ // Compute pointer to the highest-addressed slot on the frame.
+ RegI32 p = ra.needI32();
+ masm.computeEffectiveAddress(Address(sp_, localOffset(low + wordSize)),
+ p);
+
+ // Compute pointer to the lowest-addressed slot on the frame that will be
+ // initialized by the loop body.
+ RegI32 lim = ra.needI32();
+ masm.computeEffectiveAddress(Address(sp_, localOffset(loopHigh + wordSize)), lim);
+
+ // The loop body. Eventually we'll have p == lim and exit the loop.
+ Label again;
+ masm.bind(&again);
+ for (uint32_t i = 0; i < unrollLimit; ++i)
+ masm.storePtr(zero, Address(p, -(wordSize * i)));
+ masm.subPtr(Imm32(unrollLimit * wordSize), p);
+ masm.branchPtr(Assembler::LessThan, lim, p, &again);
+
+ // The tail.
+ for (uint32_t i = 0; i < tailWords; ++i)
+ masm.storePtr(zero, Address(p, -(wordSize * i)));
+
+ ra.freeI32(p);
+ ra.freeI32(lim);
+ ra.freeI32(zero);
+}
+
+// The baseline compiler proper.
+
+class BaseCompiler final : public BaseCompilerInterface
+{
+ typedef BaseStackFrame::Local Local;
+ typedef Vector<NonAssertingLabel, 8, SystemAllocPolicy> LabelVector;
+ typedef Vector<MIRType, 8, SystemAllocPolicy> MIRTypeVector;
// Bit set used for simple bounds check elimination. Capping this at 64
// locals makes sense; even 32 locals would probably be OK in practice.
//
// For more information about BCE, see the block comment above
// popMemoryAccess(), below.
typedef uint64_t BCESet;
// Control node, representing labels and stack heights at join points.
struct Control
{
Control()
- : framePushed(UINT32_MAX),
+ : stackHeight(UINT32_MAX),
stackSize(UINT32_MAX),
bceSafeOnEntry(0),
bceSafeOnExit(~BCESet(0)),
deadOnArrival(false),
deadThenBranch(false)
{}
NonAssertingLabel label; // The "exit" label
NonAssertingLabel otherLabel; // Used for the "else" branch of if-then-else
- uint32_t framePushed; // From masm
+ uint32_t stackHeight; // From BaseStackFrame
uint32_t stackSize; // Value stack height
BCESet bceSafeOnEntry; // Bounds check info flowing into the item
BCESet bceSafeOnExit; // Bounds check info flowing out of the item
bool deadOnArrival; // deadCode_ was set on entry to the region
bool deadThenBranch; // deadCode_ was set on exit from "then"
};
struct BaseCompilePolicy
@@ -898,33 +1377,33 @@ class BaseCompiler final : public BaseCo
// code density and branch prediction friendliness will be less
// important.
class OutOfLineCode : public TempObject
{
private:
NonAssertingLabel entry_;
NonAssertingLabel rejoin_;
- uint32_t framePushed_;
+ uint32_t stackHeight_;
public:
- OutOfLineCode() : framePushed_(UINT32_MAX) {}
+ OutOfLineCode() : stackHeight_(UINT32_MAX) {}
Label* entry() { return &entry_; }
Label* rejoin() { return &rejoin_; }
- void setFramePushed(uint32_t framePushed) {
- MOZ_ASSERT(framePushed_ == UINT32_MAX);
- framePushed_ = framePushed;
+ void setStackHeight(uint32_t stackHeight) {
+ MOZ_ASSERT(stackHeight_ == UINT32_MAX);
+ stackHeight_ = stackHeight;
}
- void bind(MacroAssembler& masm) {
- MOZ_ASSERT(framePushed_ != UINT32_MAX);
+ void bind(BaseStackFrame& fr, MacroAssembler& masm) {
+ MOZ_ASSERT(stackHeight_ != UINT32_MAX);
masm.bind(&entry_);
- masm.setFramePushed(framePushed_);
+ fr.setStackHeight(stackHeight_);
}
// The generate() method must be careful about register use
// because it will be invoked when there is a register
// assignment in the BaseCompiler that does not correspond
// to the available registers when the generated OOL code is
// executed. The register allocator *must not* be called.
//
@@ -966,45 +1445,41 @@ class BaseCompiler final : public BaseCo
};
const ModuleEnvironment& env_;
BaseOpIter iter_;
const FuncCompileInput& func_;
size_t lastReadCallSite_;
TempAllocator& alloc_;
const ValTypeVector& locals_; // Types of parameters and locals
- int32_t localSize_; // Size of local area in bytes (stable after beginFunction)
- int32_t varLow_; // Low byte offset of local area for true locals (not parameters)
- int32_t varHigh_; // High byte offset + 1 of local area for true locals
- int32_t maxFramePushed_; // Max value of masm.framePushed() observed
bool deadCode_; // Flag indicating we should decode & discard the opcode
bool debugEnabled_;
BCESet bceSafe_; // Locals that have been bounds checked and not updated since
ValTypeVector SigD_;
ValTypeVector SigF_;
MIRTypeVector SigP_;
MIRTypeVector SigPI_;
MIRTypeVector SigPII_;
MIRTypeVector SigPIIL_;
MIRTypeVector SigPILL_;
NonAssertingLabel returnLabel_;
NonAssertingLabel stackOverflowLabel_;
- CodeOffset stackAddOffset_;
CompileMode mode_;
LatentOp latentOp_; // Latent operation for branch (seen next)
ValType latentType_; // Operand type, if latentOp_ is true
Assembler::Condition latentIntCmp_; // Comparison operator, if latentOp_ == Compare, int types
Assembler::DoubleCondition latentDoubleCmp_;// Comparison operator, if latentOp_ == Compare, float types
FuncOffsets offsets_;
MacroAssembler& masm; // No '_' suffix - too tedious...
BaseRegAlloc ra; // Ditto
-
- Vector<Local, 8, SystemAllocPolicy> localInfo_;
+ BaseStackFrame fr;
+
+ BaseStackFrame::LocalVector localInfo_;
Vector<OutOfLineCode*, 8, SystemAllocPolicy> outOfLine_;
// On specific platforms we sometimes need to use specific registers.
#ifdef JS_CODEGEN_X64
RegI64 specific_rax;
RegI64 specific_rcx;
RegI64 specific_rdx;
@@ -1065,83 +1540,35 @@ class BaseCompiler final : public BaseCo
////////////////////////////////////////////////////////////
//
// Out of line code management.
MOZ_MUST_USE OutOfLineCode* addOutOfLineCode(OutOfLineCode* ool) {
if (!ool || !outOfLine_.append(ool))
return nullptr;
- ool->setFramePushed(masm.framePushed());
+ ool->setStackHeight(fr.stackHeight());
return ool;
}
MOZ_MUST_USE bool generateOutOfLineCode() {
for (uint32_t i = 0; i < outOfLine_.length(); i++) {
OutOfLineCode* ool = outOfLine_[i];
- ool->bind(masm);
+ ool->bind(fr, masm);
ool->generate(masm);
}
return !masm.oom();
}
- ////////////////////////////////////////////////////////////
- //
- // The stack frame.
-
- // SP-relative load and store.
-
- int32_t localOffsetToSPOffset(int32_t offset) {
- return masm.framePushed() - offset;
- }
-
- void storeToFrameI32(Register r, int32_t offset) {
- masm.store32(r, Address(StackPointer, localOffsetToSPOffset(offset)));
- }
-
- void storeToFrameI64(Register64 r, int32_t offset) {
- masm.store64(r, Address(StackPointer, localOffsetToSPOffset(offset)));
- }
-
- void storeToFramePtr(Register r, int32_t offset) {
- masm.storePtr(r, Address(StackPointer, localOffsetToSPOffset(offset)));
- }
-
- void storeToFrameF64(FloatRegister r, int32_t offset) {
- masm.storeDouble(r, Address(StackPointer, localOffsetToSPOffset(offset)));
- }
-
- void storeToFrameF32(FloatRegister r, int32_t offset) {
- masm.storeFloat32(r, Address(StackPointer, localOffsetToSPOffset(offset)));
- }
-
- void loadFromFrameI32(Register r, int32_t offset) {
- masm.load32(Address(StackPointer, localOffsetToSPOffset(offset)), r);
- }
-
- void loadFromFrameI64(Register64 r, int32_t offset) {
- masm.load64(Address(StackPointer, localOffsetToSPOffset(offset)), r);
- }
-
- void loadFromFramePtr(Register r, int32_t offset) {
- masm.loadPtr(Address(StackPointer, localOffsetToSPOffset(offset)), r);
- }
-
- void loadFromFrameF64(FloatRegister r, int32_t offset) {
- masm.loadDouble(Address(StackPointer, localOffsetToSPOffset(offset)), r);
- }
-
- void loadFromFrameF32(FloatRegister r, int32_t offset) {
- masm.loadFloat32(Address(StackPointer, localOffsetToSPOffset(offset)), r);
- }
-
- int32_t frameOffsetFromSlot(uint32_t slot, MIRType type) {
- MOZ_ASSERT(localInfo_[slot].type() == type);
- return localInfo_[slot].offs();
+ // Utility.
+
+ Local& localFromSlot(uint32_t slot, MIRType type) {
+ MOZ_ASSERT(localInfo_[slot].type == type);
+ return localInfo_[slot];
}
////////////////////////////////////////////////////////////
//
// Value stack and high-level register allocation.
//
// The value stack facilitates some on-the-fly register allocation
// and immediate-constant use. It tracks constants, latent
@@ -1250,20 +1677,26 @@ class BaseCompiler final : public BaseCo
RegF64 invalidF64() {
return RegF64(InvalidFloatReg);
}
RegI32 fromI64(RegI64 r) {
return RegI32(lowPart(r));
}
+#ifdef JS_64BIT
+ RegI64 fromI32(RegI32 r) {
+ return RegI64(Register64(r));
+ }
+#endif
+
RegI64 widenI32(RegI32 r) {
MOZ_ASSERT(!isAvailableI32(r));
#ifdef JS_PUNBOX64
- return RegI64(Register64(r));
+ return fromI32(r);
#else
RegI32 high = needI32();
return RegI64(Register64(high, r));
#endif
}
RegI32 narrowI64(RegI64 r) {
#if defined(JS_PUNBOX64)
@@ -1377,93 +1810,93 @@ class BaseCompiler final : public BaseCo
if (src != dest)
masm.moveFloat32(src, dest);
}
void setI64(int64_t v, RegI64 r) {
masm.move64(Imm64(v), r);
}
- void loadConstI32(Register r, Stk& src) {
+ void loadConstI32(RegI32 r, Stk& src) {
masm.mov(ImmWord(uint32_t(src.i32val())), r);
}
- void loadConstI32(Register r, int32_t v) {
+ void loadConstI32(RegI32 r, int32_t v) {
masm.mov(ImmWord(uint32_t(v)), r);
}
- void loadMemI32(Register r, Stk& src) {
- loadFromFrameI32(r, src.offs());
- }
-
- void loadLocalI32(Register r, Stk& src) {
- loadFromFrameI32(r, frameOffsetFromSlot(src.slot(), MIRType::Int32));
- }
-
- void loadRegisterI32(Register r, Stk& src) {
+ void loadMemI32(RegI32 r, Stk& src) {
+ fr.loadStackI32(r, src.offs());
+ }
+
+ void loadLocalI32(RegI32 r, Stk& src) {
+ fr.loadLocalI32(r, localFromSlot(src.slot(), MIRType::Int32));
+ }
+
+ void loadRegisterI32(RegI32 r, Stk& src) {
if (src.i32reg() != r)
masm.move32(src.i32reg(), r);
}
- void loadConstI64(Register64 r, Stk &src) {
+ void loadConstI64(RegI64 r, Stk &src) {
masm.move64(Imm64(src.i64val()), r);
}
- void loadMemI64(Register64 r, Stk& src) {
- loadFromFrameI64(r, src.offs());
- }
-
- void loadLocalI64(Register64 r, Stk& src) {
- loadFromFrameI64(r, frameOffsetFromSlot(src.slot(), MIRType::Int64));
- }
-
- void loadRegisterI64(Register64 r, Stk& src) {
+ void loadMemI64(RegI64 r, Stk& src) {
+ fr.loadStackI64(r, src.offs());
+ }
+
+ void loadLocalI64(RegI64 r, Stk& src) {
+ fr.loadLocalI64(r, localFromSlot(src.slot(), MIRType::Int64));
+ }
+
+ void loadRegisterI64(RegI64 r, Stk& src) {
if (src.i64reg() != r)
masm.move64(src.i64reg(), r);
}
- void loadConstF64(FloatRegister r, Stk &src) {
+ void loadConstF64(RegF64 r, Stk &src) {
double d;
src.f64val(&d);
masm.loadConstantDouble(d, r);
}
- void loadMemF64(FloatRegister r, Stk& src) {
- loadFromFrameF64(r, src.offs());
- }
-
- void loadLocalF64(FloatRegister r, Stk& src) {
- loadFromFrameF64(r, frameOffsetFromSlot(src.slot(), MIRType::Double));
- }
-
- void loadRegisterF64(FloatRegister r, Stk& src) {
+ void loadMemF64(RegF64 r, Stk& src) {
+ fr.loadStackF64(r, src.offs());
+ }
+
+ void loadLocalF64(RegF64 r, Stk& src) {
+ fr.loadLocalF64(r, localFromSlot(src.slot(), MIRType::Double));
+ }
+
+ void loadRegisterF64(RegF64 r, Stk& src) {
if (src.f64reg() != r)
masm.moveDouble(src.f64reg(), r);
}
- void loadConstF32(FloatRegister r, Stk &src) {
+ void loadConstF32(RegF32 r, Stk &src) {
float f;
src.f32val(&f);
masm.loadConstantFloat32(f, r);
}
- void loadMemF32(FloatRegister r, Stk& src) {
- loadFromFrameF32(r, src.offs());
- }
-
- void loadLocalF32(FloatRegister r, Stk& src) {
- loadFromFrameF32(r, frameOffsetFromSlot(src.slot(), MIRType::Float32));
- }
-
- void loadRegisterF32(FloatRegister r, Stk& src) {
+ void loadMemF32(RegF32 r, Stk& src) {
+ fr.loadStackF32(r, src.offs());
+ }
+
+ void loadLocalF32(RegF32 r, Stk& src) {
+ fr.loadLocalF32(r, localFromSlot(src.slot(), MIRType::Float32));
+ }
+
+ void loadRegisterF32(RegF32 r, Stk& src) {
if (src.f32reg() != r)
masm.moveFloat32(src.f32reg(), r);
}
- void loadI32(Register r, Stk& src) {
+ void loadI32(RegI32 r, Stk& src) {
switch (src.kind()) {
case Stk::ConstI32:
loadConstI32(r, src);
break;
case Stk::MemI32:
loadMemI32(r, src);
break;
case Stk::LocalI32:
@@ -1473,17 +1906,17 @@ class BaseCompiler final : public BaseCo
loadRegisterI32(r, src);
break;
case Stk::None:
default:
MOZ_CRASH("Compiler bug: Expected I32 on stack");
}
}
- void loadI64(Register64 r, Stk& src) {
+ void loadI64(RegI64 r, Stk& src) {
switch (src.kind()) {
case Stk::ConstI64:
loadConstI64(r, src);
break;
case Stk::MemI64:
loadMemI64(r, src);
break;
case Stk::LocalI64:
@@ -1494,60 +1927,60 @@ class BaseCompiler final : public BaseCo
break;
case Stk::None:
default:
MOZ_CRASH("Compiler bug: Expected I64 on stack");
}
}
#if !defined(JS_PUNBOX64)
- void loadI64Low(Register r, Stk& src) {
+ void loadI64Low(RegI32 r, Stk& src) {
switch (src.kind()) {
case Stk::ConstI64:
masm.move32(Imm64(src.i64val()).low(), r);
break;
case Stk::MemI64:
- loadFromFrameI32(r, src.offs() - INT64LOW_OFFSET);
+ fr.loadStackI64Low(r, src.offs());
break;
case Stk::LocalI64:
- loadFromFrameI32(r, frameOffsetFromSlot(src.slot(), MIRType::Int64) - INT64LOW_OFFSET);
+ fr.loadLocalI64Low(r, localFromSlot(src.slot(), MIRType::Int64));
break;
case Stk::RegisterI64:
if (src.i64reg().low != r)
masm.move32(src.i64reg().low, r);
break;
case Stk::None:
default:
MOZ_CRASH("Compiler bug: Expected I64 on stack");
}
}
- void loadI64High(Register r, Stk& src) {
+ void loadI64High(RegI32 r, Stk& src) {
switch (src.kind()) {
case Stk::ConstI64:
masm.move32(Imm64(src.i64val()).hi(), r);
break;
case Stk::MemI64:
- loadFromFrameI32(r, src.offs() - INT64HIGH_OFFSET);
+ fr.loadStackI64High(r, src.offs());
break;
case Stk::LocalI64:
- loadFromFrameI32(r, frameOffsetFromSlot(src.slot(), MIRType::Int64) - INT64HIGH_OFFSET);
+ fr.loadLocalI64High(r, localFromSlot(src.slot(), MIRType::Int64));
break;
case Stk::RegisterI64:
if (src.i64reg().high != r)
masm.move32(src.i64reg().high, r);
break;
case Stk::None:
default:
MOZ_CRASH("Compiler bug: Expected I64 on stack");
}
}
#endif
- void loadF64(FloatRegister r, Stk& src) {
+ void loadF64(RegF64 r, Stk& src) {
switch (src.kind()) {
case Stk::ConstF64:
loadConstF64(r, src);
break;
case Stk::MemF64:
loadMemF64(r, src);
break;
case Stk::LocalF64:
@@ -1557,17 +1990,17 @@ class BaseCompiler final : public BaseCo
loadRegisterF64(r, src);
break;
case Stk::None:
default:
MOZ_CRASH("Compiler bug: expected F64 on stack");
}
}
- void loadF32(FloatRegister r, Stk& src) {
+ void loadF32(RegF32 r, Stk& src) {
switch (src.kind()) {
case Stk::ConstF32:
loadConstF32(r, src);
break;
case Stk::MemF32:
loadMemF32(r, src);
break;
case Stk::LocalF32:
@@ -1617,86 +2050,83 @@ class BaseCompiler final : public BaseCo
}
for (size_t i = start; i < lim; i++) {
Stk& v = stk_[i];
switch (v.kind()) {
case Stk::LocalI32: {
ScratchI32 scratch(*this);
loadLocalI32(scratch, v);
- masm.Push(scratch);
- v.setOffs(Stk::MemI32, masm.framePushed());
+ uint32_t offs = fr.pushPtr(scratch);
+ v.setOffs(Stk::MemI32, offs);
break;
}
case Stk::RegisterI32: {
- masm.Push(v.i32reg());
+ uint32_t offs = fr.pushPtr(v.i32reg());
freeI32(v.i32reg());
- v.setOffs(Stk::MemI32, masm.framePushed());
+ v.setOffs(Stk::MemI32, offs);
break;
}
case Stk::LocalI64: {
ScratchI32 scratch(*this);
#ifdef JS_PUNBOX64
- loadI64(Register64(scratch), v);
- masm.Push(scratch);
+ loadI64(fromI32(scratch), v);
+ uint32_t offs = fr.pushPtr(scratch);
#else
- int32_t offset = frameOffsetFromSlot(v.slot(), MIRType::Int64);
- loadFromFrameI32(scratch, offset - INT64HIGH_OFFSET);
- masm.Push(scratch);
- loadFromFrameI32(scratch, offset - INT64LOW_OFFSET);
- masm.Push(scratch);
-#endif
- v.setOffs(Stk::MemI64, masm.framePushed());
+ fr.loadLocalI64High(scratch, localFromSlot(v.slot(), MIRType::Int64));
+ fr.pushPtr(scratch);
+ fr.loadLocalI64Low(scratch, localFromSlot(v.slot(), MIRType::Int64));
+ uint32_t offs = fr.pushPtr(scratch);
+#endif
+ v.setOffs(Stk::MemI64, offs);
break;
}
case Stk::RegisterI64: {
#ifdef JS_PUNBOX64
- masm.Push(v.i64reg().reg);
+ uint32_t offs = fr.pushPtr(v.i64reg().reg);
freeI64(v.i64reg());
#else
- masm.Push(v.i64reg().high);
- masm.Push(v.i64reg().low);
+ fr.pushPtr(v.i64reg().high);
+ uint32_t offs = fr.pushPtr(v.i64reg().low);
freeI64(v.i64reg());
#endif
- v.setOffs(Stk::MemI64, masm.framePushed());
+ v.setOffs(Stk::MemI64, offs);
break;
}
case Stk::LocalF64: {
ScratchF64 scratch(*this);
loadF64(scratch, v);
- masm.Push(scratch);
- v.setOffs(Stk::MemF64, masm.framePushed());
+ uint32_t offs = fr.pushDouble(scratch);
+ v.setOffs(Stk::MemF64, offs);
break;
}
case Stk::RegisterF64: {
- masm.Push(v.f64reg());
+ uint32_t offs = fr.pushDouble(v.f64reg());
freeF64(v.f64reg());
- v.setOffs(Stk::MemF64, masm.framePushed());
+ v.setOffs(Stk::MemF64, offs);
break;
}
case Stk::LocalF32: {
ScratchF32 scratch(*this);
loadF32(scratch, v);
- masm.Push(scratch);
- v.setOffs(Stk::MemF32, masm.framePushed());
+ uint32_t offs = fr.pushFloat(scratch);
+ v.setOffs(Stk::MemF32, offs);
break;
}
case Stk::RegisterF32: {
- masm.Push(v.f32reg());
+ uint32_t offs = fr.pushFloat(v.f32reg());
freeF32(v.f32reg());
- v.setOffs(Stk::MemF32, masm.framePushed());
+ v.setOffs(Stk::MemF32, offs);
break;
}
default: {
break;
}
}
}
-
- maxFramePushed_ = Max(maxFramePushed_, int32_t(masm.framePushed()));
}
// This is an optimization used to avoid calling sync() for
// setLocal(): if the local does not exist unresolved on the stack
// then we can skip the sync.
bool hasLocal(uint32_t slot) {
for (size_t i = stk_.length(); i > 0; i--) {
@@ -1797,17 +2227,17 @@ class BaseCompiler final : public BaseCo
switch (v.kind()) {
case Stk::ConstI32:
loadConstI32(r, v);
break;
case Stk::LocalI32:
loadLocalI32(r, v);
break;
case Stk::MemI32:
- masm.Pop(r);
+ fr.popPtr(r);
break;
case Stk::RegisterI32:
loadRegisterI32(r, v);
break;
case Stk::None:
default:
MOZ_CRASH("Compiler bug: expected int on stack");
}
@@ -1846,20 +2276,20 @@ class BaseCompiler final : public BaseCo
case Stk::ConstI64:
loadConstI64(r, v);
break;
case Stk::LocalI64:
loadLocalI64(r, v);
break;
case Stk::MemI64:
#ifdef JS_PUNBOX64
- masm.Pop(r.reg);
+ fr.popPtr(r.reg);
#else
- masm.Pop(r.low);
- masm.Pop(r.high);
+ fr.popPtr(r.low);
+ fr.popPtr(r.high);
#endif
break;
case Stk::RegisterI64:
loadRegisterI64(r, v);
break;
case Stk::None:
default:
MOZ_CRASH("Compiler bug: expected long on stack");
@@ -1903,17 +2333,17 @@ class BaseCompiler final : public BaseCo
switch (v.kind()) {
case Stk::ConstF64:
loadConstF64(r, v);
break;
case Stk::LocalF64:
loadLocalF64(r, v);
break;
case Stk::MemF64:
- masm.Pop(r);
+ fr.popDouble(r);
break;
case Stk::RegisterF64:
loadRegisterF64(r, v);
break;
case Stk::None:
default:
MOZ_CRASH("Compiler bug: expected double on stack");
}
@@ -1951,17 +2381,17 @@ class BaseCompiler final : public BaseCo
switch (v.kind()) {
case Stk::ConstF32:
loadConstF32(r, v);
break;
case Stk::LocalF32:
loadLocalF32(r, v);
break;
case Stk::MemF32:
- masm.Pop(r);
+ fr.popFloat(r);
break;
case Stk::RegisterF32:
loadRegisterF32(r, v);
break;
case Stk::None:
default:
MOZ_CRASH("Compiler bug: expected float on stack");
}
@@ -2241,56 +2671,23 @@ class BaseCompiler final : public BaseCo
// Return the amount of execution stack consumed by the top numval
// values on the value stack.
size_t stackConsumed(size_t numval) {
size_t size = 0;
MOZ_ASSERT(numval <= stk_.length());
for (uint32_t i = stk_.length() - 1; numval > 0; numval--, i--) {
- // The size computations come from the implementation of Push() in
- // MacroAssembler-x86-shared.cpp and MacroAssembler-arm-shared.cpp,
- // and from VFPRegister::size() in Architecture-arm.h.
- //
- // On ARM unlike on x86 we push a single for float.
-
Stk& v = stk_[i];
switch (v.kind()) {
- case Stk::MemI32:
-#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
- size += sizeof(intptr_t);
-#else
- MOZ_CRASH("BaseCompiler platform hook: stackConsumed I32");
-#endif
- break;
- case Stk::MemI64:
-#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
- size += sizeof(int64_t);
-#else
- MOZ_CRASH("BaseCompiler platform hook: stackConsumed I64");
-#endif
- break;
- case Stk::MemF64:
-#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
- size += sizeof(double);
-#else
- MOZ_CRASH("BaseCompiler platform hook: stackConsumed F64");
-#endif
- break;
- case Stk::MemF32:
-#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
- size += sizeof(double);
-#elif defined(JS_CODEGEN_ARM)
- size += sizeof(float);
-#else
- MOZ_CRASH("BaseCompiler platform hook: stackConsumed F32");
-#endif
- break;
- default:
- break;
+ case Stk::MemI32: size += BaseStackFrame::StackSizeOfPtr; break;
+ case Stk::MemI64: size += BaseStackFrame::StackSizeOfInt64; break;
+ case Stk::MemF64: size += BaseStackFrame::StackSizeOfDouble; break;
+ case Stk::MemF32: size += BaseStackFrame::StackSizeOfFloat; break;
+ default: break;
}
}
return size;
}
void popValueStackTo(uint32_t stackSize) {
for (uint32_t i = stk_.length(); i > stackSize; i--) {
Stk& v = stk_[i-1];
@@ -2313,48 +2710,20 @@ class BaseCompiler final : public BaseCo
}
stk_.shrinkTo(stackSize);
}
void popValueStackBy(uint32_t items) {
popValueStackTo(stk_.length() - items);
}
- // Before branching to an outer control label, pop the execution
- // stack to the level expected by that region, but do not free the
- // stack as that will happen as compilation leaves the block.
-
- void popStackBeforeBranch(uint32_t framePushed) {
- uint32_t frameHere = masm.framePushed();
- if (frameHere > framePushed)
- masm.addPtr(ImmWord(frameHere - framePushed), StackPointer);
- }
-
- bool willPopStackBeforeBranch(uint32_t framePushed) {
- uint32_t frameHere = masm.framePushed();
- return frameHere > framePushed;
- }
-
- // Before exiting a nested control region, pop the execution stack
- // to the level expected by the nesting region, and free the
- // stack.
-
- void popStackOnBlockExit(uint32_t framePushed) {
- uint32_t frameHere = masm.framePushed();
- if (frameHere > framePushed) {
- if (deadCode_)
- masm.adjustStack(frameHere - framePushed);
- else
- masm.freeStack(frameHere - framePushed);
- }
- }
-
- void popStackIfMemory() {
+ void dropValue() {
if (peek(0).isMem())
- masm.freeStack(stackConsumed(1));
+ fr.popBytes(stackConsumed(1));
+ popValueStackBy(1);
}
// Peek at the stack, for calls.
Stk& peek(uint32_t relativeDepth) {
return stk_[stk_.length()-1-relativeDepth];
}
@@ -2390,19 +2759,19 @@ class BaseCompiler final : public BaseCo
////////////////////////////////////////////////////////////
//
// Control stack
void initControl(Control& item)
{
// Make sure the constructor was run properly
- MOZ_ASSERT(item.framePushed == UINT32_MAX && item.stackSize == UINT32_MAX);
-
- item.framePushed = masm.framePushed();
+ MOZ_ASSERT(item.stackHeight == UINT32_MAX && item.stackSize == UINT32_MAX);
+
+ item.stackHeight = fr.stackHeight();
item.stackSize = stk_.length();
item.deadOnArrival = deadCode_;
item.bceSafeOnEntry = bceSafe_;
}
Control& controlItem() {
return iter_.controlItem();
}
@@ -2430,89 +2799,77 @@ class BaseCompiler final : public BaseCo
//
// Function prologue and epilogue.
void beginFunction() {
JitSpew(JitSpew_Codegen, "# Emitting wasm baseline code");
SigIdDesc sigId = env_.funcSigs[func_.index]->id;
if (mode_ == CompileMode::Tier1)
- GenerateFunctionPrologue(masm, localSize_, sigId, &offsets_, mode_, func_.index);
+ GenerateFunctionPrologue(masm, fr.initialSize(), sigId, &offsets_, mode_, func_.index);
else
- GenerateFunctionPrologue(masm, localSize_, sigId, &offsets_);
-
- MOZ_ASSERT(masm.framePushed() == uint32_t(localSize_));
-
- maxFramePushed_ = localSize_;
+ GenerateFunctionPrologue(masm, fr.initialSize(), sigId, &offsets_);
+
+ fr.endFunctionPrologue();
if (debugEnabled_) {
// Initialize funcIndex and flag fields of DebugFrame.
size_t debugFrame = masm.framePushed() - DebugFrame::offsetOfFrame();
masm.store32(Imm32(func_.index),
Address(masm.getStackPointer(), debugFrame + DebugFrame::offsetOfFuncIndex()));
masm.storePtr(ImmWord(0),
Address(masm.getStackPointer(), debugFrame + DebugFrame::offsetOfFlagsWord()));
}
- // We won't know until after we've generated code how big the frame will
- // be (we may need arbitrary spill slots and outgoing param slots) so
- // emit a patchable add that is patched in endFunction().
- //
- // ScratchReg may be used by branchPtr(), so use ABINonArgReg0/1 for
- // temporaries.
-
- stackAddOffset_ = masm.add32ToPtrWithPatch(StackPointer, ABINonArgReg0);
- masm.wasmEmitStackCheck(ABINonArgReg0, ABINonArgReg1, &stackOverflowLabel_);
+ fr.allocStack(&stackOverflowLabel_);
// Copy arguments from registers to stack.
const ValTypeVector& args = sig().args();
for (ABIArgIter<const ValTypeVector> i(args); !i.done(); i++) {
Local& l = localInfo_[i.index()];
switch (i.mirType()) {
case MIRType::Int32:
if (i->argInRegister())
- storeToFrameI32(i->gpr(), l.offs());
+ fr.storeLocalI32(RegI32(i->gpr()), l);
break;
case MIRType::Int64:
if (i->argInRegister())
- storeToFrameI64(i->gpr64(), l.offs());
+ fr.storeLocalI64(RegI64(i->gpr64()), l);
break;
case MIRType::Double:
if (i->argInRegister())
- storeToFrameF64(i->fpu(), l.offs());
+ fr.storeLocalF64(RegF64(i->fpu()), l);
break;
case MIRType::Float32:
if (i->argInRegister())
- storeToFrameF32(i->fpu(), l.offs());
+ fr.storeLocalF32(RegF32(i->fpu()), l);
break;
default:
MOZ_CRASH("Function argument type");
}
}
- if (varLow_ < varHigh_)
- emitInitStackLocals();
+ fr.zeroLocals(ra);
if (debugEnabled_)
insertBreakablePoint(CallSiteDesc::EnterFrame);
}
void saveResult() {
MOZ_ASSERT(debugEnabled_);
size_t debugFrameOffset = masm.framePushed() - DebugFrame::offsetOfFrame();
- Address resultsAddress(StackPointer, debugFrameOffset + DebugFrame::offsetOfResults());
+ Address resultsAddress(masm.getStackPointer(), debugFrameOffset + DebugFrame::offsetOfResults());
switch (sig().ret()) {
case ExprType::Void:
break;
case ExprType::I32:
masm.store32(RegI32(ReturnReg), resultsAddress);
break;
-
case ExprType::I64:
masm.store64(RegI64(ReturnReg64), resultsAddress);
break;
case ExprType::F64:
masm.storeDouble(RegF64(ReturnDoubleReg), resultsAddress);
break;
case ExprType::F32:
masm.storeFloat32(RegF32(ReturnFloat32Reg), resultsAddress);
@@ -2520,17 +2877,17 @@ class BaseCompiler final : public BaseCo
default:
MOZ_CRASH("Function return type");
}
}
void restoreResult() {
MOZ_ASSERT(debugEnabled_);
size_t debugFrameOffset = masm.framePushed() - DebugFrame::offsetOfFrame();
- Address resultsAddress(StackPointer, debugFrameOffset + DebugFrame::offsetOfResults());
+ Address resultsAddress(masm.getStackPointer(), debugFrameOffset + DebugFrame::offsetOfResults());
switch (sig().ret()) {
case ExprType::Void:
break;
case ExprType::I32:
masm.load32(resultsAddress, RegI32(ReturnReg));
break;
case ExprType::I64:
masm.load64(resultsAddress, RegI64(ReturnReg64));
@@ -2547,69 +2904,66 @@ class BaseCompiler final : public BaseCo
}
bool endFunction() {
// Always branch to stackOverflowLabel_ or returnLabel_.
masm.breakpoint();
// Patch the add in the prologue so that it checks against the correct
// frame size. Flush the constant pool in case it needs to be patched.
- MOZ_ASSERT(maxFramePushed_ >= localSize_);
masm.flush();
// Precondition for patching.
if (masm.oom())
return false;
- masm.patchAdd32ToPtr(stackAddOffset_, Imm32(-int32_t(maxFramePushed_ - localSize_)));
+
+ fr.patchAllocStack();
// Since we just overflowed the stack, to be on the safe side, pop the
// stack so that, when the trap exit stub executes, it is a safe
// distance away from the end of the native stack. If debugEnabled_ is
// set, we pop all locals space except allocated for DebugFrame to
// maintain the invariant that, when debugEnabled_, all wasm::Frames
// are valid wasm::DebugFrames which is observable by WasmHandleThrow.
masm.bind(&stackOverflowLabel_);
- int32_t debugFrameReserved = debugEnabled_ ? DebugFrame::offsetOfFrame() : 0;
- MOZ_ASSERT(localSize_ >= debugFrameReserved);
- if (localSize_ > debugFrameReserved)
- masm.addToStackPtr(Imm32(localSize_ - debugFrameReserved));
+ uint32_t debugFrameReserved = debugEnabled_ ? DebugFrame::offsetOfFrame() : 0;
+ MOZ_ASSERT(fr.initialSize() >= debugFrameReserved);
+ if (fr.initialSize() > debugFrameReserved)
+ masm.addToStackPtr(Imm32(fr.initialSize() - debugFrameReserved));
BytecodeOffset prologueTrapOffset(func_.lineOrBytecode);
masm.jump(TrapDesc(prologueTrapOffset, Trap::StackOverflow, debugFrameReserved));
masm.bind(&returnLabel_);
if (debugEnabled_) {
// Store and reload the return value from DebugFrame::return so that
// it can be clobbered, and/or modified by the debug trap.
saveResult();
insertBreakablePoint(CallSiteDesc::Breakpoint);
insertBreakablePoint(CallSiteDesc::LeaveFrame);
restoreResult();
}
- GenerateFunctionEpilogue(masm, localSize_, &offsets_);
+ GenerateFunctionEpilogue(masm, fr.initialSize(), &offsets_);
#if defined(JS_ION_PERF)
// FIXME - profiling code missing. No bug for this.
// Note the end of the inline code and start of the OOL code.
//gen->perfSpewer().noteEndInlineCode(masm);
#endif
if (!generateOutOfLineCode())
return false;
masm.wasmEmitTrapOutOfLineCode();
offsets_.end = masm.currentOffset();
- // A frame greater than 256KB is implausible, probably an attack,
- // so fail the compilation.
-
- if (maxFramePushed_ > 256 * 1024)
+ if (!fr.checkStackHeight())
return false;
return !masm.oom();
}
//////////////////////////////////////////////////////////////////////
//
// Calls.
@@ -2652,24 +3006,27 @@ class BaseCompiler final : public BaseCo
call.hardFP = true;
# else
call.hardFP = false;
# endif
call.abi.setUseHardFp(call.hardFP);
#endif
}
+ // Use masm.framePushed() because the value we want here does not depend
+ // on the height of the frame's stack area, but the actual size of the
+ // allocated frame.
call.frameAlignAdjustment = ComputeByteAlignment(masm.framePushed() + sizeof(Frame),
JitStackAlignment);
}
void endCall(FunctionCall& call, size_t stackSpace)
{
size_t adjustment = call.stackArgAreaSize + call.frameAlignAdjustment;
- masm.freeStack(stackSpace + adjustment);
+ fr.freeArgAreaAndPopBytes(adjustment, stackSpace);
if (call.reloadMachineStateAfter) {
// On x86 there are no pinned registers, so don't waste time
// reloading the Tls.
#ifndef JS_CODEGEN_X86
masm.loadWasmTlsRegFromFrame();
masm.loadWasmPinnedRegsFromTls();
#endif
@@ -2691,18 +3048,17 @@ class BaseCompiler final : public BaseCo
return AlignBytes(i.stackBytesConsumedSoFar(), 16u);
}
void startCallArgs(FunctionCall& call, size_t stackArgAreaSize)
{
call.stackArgAreaSize = stackArgAreaSize;
size_t adjustment = call.stackArgAreaSize + call.frameAlignAdjustment;
- if (adjustment)
- masm.reserveStack(adjustment);
+ fr.allocArgArea(adjustment);
}
const ABIArg reservePointerArgument(FunctionCall& call) {
return call.abi.next(MIRType::Pointer);
}
// TODO / OPTIMIZE (Bug 1316821): Note passArg is used only in one place.
// (Or it was, until Luke wandered through, but that can be fixed again.)
@@ -2727,92 +3083,92 @@ class BaseCompiler final : public BaseCo
void passArg(FunctionCall& call, ValType type, Stk& arg) {
switch (type) {
case ValType::I32: {
ABIArg argLoc = call.abi.next(MIRType::Int32);
if (argLoc.kind() == ABIArg::Stack) {
ScratchI32 scratch(*this);
loadI32(scratch, arg);
- masm.store32(scratch, Address(StackPointer, argLoc.offsetFromArgBase()));
+ masm.store32(scratch, Address(masm.getStackPointer(), argLoc.offsetFromArgBase()));
} else {
- loadI32(argLoc.gpr(), arg);
+ loadI32(RegI32(argLoc.gpr()), arg);
}
break;
}
case ValType::I64: {
ABIArg argLoc = call.abi.next(MIRType::Int64);
if (argLoc.kind() == ABIArg::Stack) {
ScratchI32 scratch(*this);
#if defined(JS_CODEGEN_X64)
- loadI64(Register64(scratch), arg);
- masm.movq(scratch, Operand(StackPointer, argLoc.offsetFromArgBase()));
+ loadI64(fromI32(scratch), arg);
+ masm.movq(scratch, Operand(masm.getStackPointer(), argLoc.offsetFromArgBase()));
#elif defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
loadI64Low(scratch, arg);
- masm.store32(scratch, LowWord(Address(StackPointer, argLoc.offsetFromArgBase())));
+ masm.store32(scratch, LowWord(Address(masm.getStackPointer(), argLoc.offsetFromArgBase())));
loadI64High(scratch, arg);
- masm.store32(scratch, HighWord(Address(StackPointer, argLoc.offsetFromArgBase())));
+ masm.store32(scratch, HighWord(Address(masm.getStackPointer(), argLoc.offsetFromArgBase())));
#else
MOZ_CRASH("BaseCompiler platform hook: passArg I64");
#endif
} else {
- loadI64(argLoc.gpr64(), arg);
+ loadI64(RegI64(argLoc.gpr64()), arg);
}
break;
}
case ValType::F64: {
ABIArg argLoc = call.abi.next(MIRType::Double);
switch (argLoc.kind()) {
case ABIArg::Stack: {
ScratchF64 scratch(*this);
loadF64(scratch, arg);
- masm.storeDouble(scratch, Address(StackPointer, argLoc.offsetFromArgBase()));
+ masm.storeDouble(scratch, Address(masm.getStackPointer(), argLoc.offsetFromArgBase()));
break;
}
#if defined(JS_CODEGEN_REGISTER_PAIR)
case ABIArg::GPR_PAIR: {
# ifdef JS_CODEGEN_ARM
ScratchF64 scratch(*this);
loadF64(scratch, arg);
masm.ma_vxfer(scratch, argLoc.evenGpr(), argLoc.oddGpr());
break;
# else
MOZ_CRASH("BaseCompiler platform hook: passArg F64 pair");
# endif
}
#endif
case ABIArg::FPU: {
- loadF64(argLoc.fpu(), arg);
+ loadF64(RegF64(argLoc.fpu()), arg);
break;
}
case ABIArg::GPR: {
MOZ_CRASH("Unexpected parameter passing discipline");
}
case ABIArg::Uninitialized:
MOZ_CRASH("Uninitialized ABIArg kind");
}
break;
}
case ValType::F32: {
ABIArg argLoc = call.abi.next(MIRType::Float32);
switch (argLoc.kind()) {
case ABIArg::Stack: {
ScratchF32 scratch(*this);
loadF32(scratch, arg);
- masm.storeFloat32(scratch, Address(StackPointer, argLoc.offsetFromArgBase()));
+ masm.storeFloat32(scratch, Address(masm.getStackPointer(), argLoc.offsetFromArgBase()));
break;
}
case ABIArg::GPR: {
ScratchF32 scratch(*this);
loadF32(scratch, arg);
masm.moveFloat32ToGPR(scratch, argLoc.gpr());
break;
}
case ABIArg::FPU: {
- loadF32(argLoc.fpu(), arg);
+ loadF32(RegF32(argLoc.fpu()), arg);
break;
}
#if defined(JS_CODEGEN_REGISTER_PAIR)
case ABIArg::GPR_PAIR: {
MOZ_CRASH("Unexpected parameter passing discipline");
}
#endif
case ABIArg::Uninitialized:
@@ -2841,17 +3197,17 @@ class BaseCompiler final : public BaseCo
void callIndirect(uint32_t sigIndex, Stk& indexVal, const FunctionCall& call)
{
const SigWithId& sig = env_.sigs[sigIndex];
MOZ_ASSERT(sig.id.kind() != SigIdDesc::Kind::None);
MOZ_ASSERT(env_.tables.length() == 1);
const TableDesc& table = env_.tables[0];
- loadI32(WasmTableCallIndexReg, indexVal);
+ loadI32(RegI32(WasmTableCallIndexReg), indexVal);
CallSiteDesc desc(call.lineOrBytecode, CallSiteDesc::Dynamic);
CalleeDesc callee = CalleeDesc::wasmTable(table, sig.id);
masm.wasmCallIndirect(desc, callee, NeedsBoundsCheck(true));
}
// Precondition: sync()
@@ -2983,17 +3339,17 @@ class BaseCompiler final : public BaseCo
if (call.usesSystemAbi && !call.hardFP)
masm.ma_vxfer(r0, r1, rv);
#endif
return rv;
}
void returnCleanup(bool popStack) {
if (popStack)
- popStackBeforeBranch(controlOutermost().framePushed);
+ fr.popStackBeforeBranch(controlOutermost().stackHeight);
masm.jump(&returnLabel_);
}
void pop2xI32ForIntMulDiv(RegI32* r0, RegI32* r1) {
#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
// srcDest must be eax, and edx will be clobbered.
need2xI32(specific_eax, specific_edx);
*r1 = popI32();
@@ -3164,30 +3520,30 @@ class BaseCompiler final : public BaseCo
MOZ_CRASH("BaseCompiler platform hook: popcnt64NeedsTemp");
#endif
}
RegI64 popI32ForSignExtendI64() {
#if defined(JS_CODEGEN_X86)
need2xI32(specific_edx, specific_eax);
RegI32 r0 = popI32ToSpecific(specific_eax);
- RegI64 x0 = RegI64(Register64(specific_edx, specific_eax));
+ RegI64 x0 = specific_edx_eax;
(void)r0; // x0 is the widening of r0
#else
RegI32 r0 = popI32();
RegI64 x0 = widenI32(r0);
#endif
return x0;
}
RegI64 popI64ForSignExtendI64() {
#if defined(JS_CODEGEN_X86)
need2xI32(specific_edx, specific_eax);
// Low on top, high underneath
- return popI64ToSpecific(RegI64(Register64(specific_edx, specific_eax)));
+ return popI64ToSpecific(specific_edx_eax);
#else
return popI64();
#endif
}
class OutOfLineTruncateF32OrF64ToI32 : public OutOfLineCode
{
AnyReg src;
@@ -3843,17 +4199,17 @@ class BaseCompiler final : public BaseCo
case Scalar::Uint8: {
RegI32 v = rv;
RegI32 d = rd;
#ifdef JS_CODEGEN_X86
// The temp, if used, must be a byte register.
MOZ_ASSERT(tmp == invalidI32());
ScratchEBX scratch(*this);
if (op != AtomicFetchAddOp && op != AtomicFetchSubOp)
- tmp = RegI32(scratch);
+ tmp = scratch;
#endif
switch (op) {
case AtomicFetchAddOp: masm.atomicFetchAdd8ZeroExtend(v, srcAddr, tmp, d); break;
case AtomicFetchSubOp: masm.atomicFetchSub8ZeroExtend(v, srcAddr, tmp, d); break;
case AtomicFetchAndOp: masm.atomicFetchAnd8ZeroExtend(v, srcAddr, tmp, d); break;
case AtomicFetchOrOp: masm.atomicFetchOr8ZeroExtend(v, srcAddr, tmp, d); break;
case AtomicFetchXorOp: masm.atomicFetchXor8ZeroExtend(v, srcAddr, tmp, d); break;
default: MOZ_CRASH("No such op");
@@ -3927,17 +4283,17 @@ class BaseCompiler final : public BaseCo
switch (access->type()) {
case Scalar::Uint8: {
#if defined(JS_CODEGEN_X86)
ScratchEBX scratch(*this);
MOZ_ASSERT(rd == specific_eax);
if (!ra.isSingleByteI32(rnew)) {
// The replacement value must have a byte persona.
masm.movl(rnew, scratch);
- rnew = RegI32(scratch);
+ rnew = scratch;
}
#endif
masm.compareExchange8ZeroExtend(srcAddr, rexpect, rnew, rd);
break;
}
case Scalar::Uint16:
masm.compareExchange16ZeroExtend(srcAddr, rexpect, rnew, rd);
break;
@@ -4069,16 +4425,19 @@ class BaseCompiler final : public BaseCo
return iter_.done();
}
BytecodeOffset bytecodeOffset() const {
return iter_.bytecodeOffset();
}
TrapDesc trap(Trap t) const {
+ // Use masm.framePushed() because the value needed by the trap machinery
+ // is the size of the frame overall, not the height of the stack area of
+ // the frame.
return TrapDesc(bytecodeOffset(), t, masm.framePushed());
}
////////////////////////////////////////////////////////////
//
// Machinery for optimized conditional branches.
//
// To disable this optimization it is enough always to return false from
@@ -4106,24 +4465,24 @@ class BaseCompiler final : public BaseCo
} f32;
struct {
RegF64 lhs;
RegF64 rhs;
} f64;
};
Label* const label; // The target of the branch, never NULL
- const int32_t framePushed; // Either NoPop, or the value to pop to along the taken edge
+ const int32_t stackHeight; // Either NoPop, or the value to pop to along the taken edge
const bool invertBranch; // If true, invert the sense of the branch
const ExprType resultType; // The result propagated along the edges, or Void
- explicit BranchState(Label* label, int32_t framePushed = NoPop,
+ explicit BranchState(Label* label, int32_t stackHeight = NoPop,
uint32_t invertBranch = false, ExprType resultType = ExprType::Void)
: label(label),
- framePushed(framePushed),
+ stackHeight(stackHeight),
invertBranch(invertBranch),
resultType(resultType)
{}
};
void setLatentCompare(Assembler::Condition compareOp, ValType operandType) {
latentOp_ = LatentOp::Compare;
latentType_ = operandType;
@@ -4169,30 +4528,30 @@ class BaseCompiler final : public BaseCo
masm.branch64(c, lhs, rhs, l);
}
// Emit a conditional branch that optionally and optimally cleans up the CPU
// stack before we branch.
//
// Cond is either Assembler::Condition or Assembler::DoubleCondition.
//
- // Lhs is Register, Register64, or FloatRegister.
+ // Lhs is RegI32, RegI64, or RegF32, or RegF64.
//
// Rhs is either the same as Lhs, or an immediate expression compatible with
// Lhs "when applicable".
template<typename Cond, typename Lhs, typename Rhs>
void jumpConditionalWithJoinReg(BranchState* b, Cond cond, Lhs lhs, Rhs rhs)
{
Maybe<AnyReg> r = popJoinRegUnlessVoid(b->resultType);
- if (b->framePushed != BranchState::NoPop && willPopStackBeforeBranch(b->framePushed)) {
+ if (b->stackHeight != BranchState::NoPop && fr.willPopStackBeforeBranch(b->stackHeight)) {
Label notTaken;
branchTo(b->invertBranch ? cond : Assembler::InvertCondition(cond), lhs, rhs, ¬Taken);
- popStackBeforeBranch(b->framePushed);
+ fr.popStackBeforeBranch(b->stackHeight);
masm.jump(b->label);
masm.bind(¬Taken);
} else {
branchTo(b->invertBranch ? Assembler::InvertCondition(cond) : cond, lhs, rhs, b->label);
}
pushJoinRegUnlessVoid(r);
}
@@ -4495,17 +4854,17 @@ BaseCompiler::emitMultiplyI64()
// srcDest must be rax, and rdx will be clobbered.
need2xI64(specific_rax, specific_rdx);
r1 = popI64();
r0 = popI64ToSpecific(specific_rax);
freeI64(specific_rdx);
#elif defined(JS_CODEGEN_X86)
need2xI32(specific_eax, specific_edx);
r1 = popI64();
- r0 = popI64ToSpecific(RegI64(Register64(specific_edx, specific_eax)));
+ r0 = popI64ToSpecific(specific_edx_eax);
temp = needI32();
#else
pop2xI64(&r0, &r1);
temp = needI32();
#endif
masm.mul64(r1, r0, temp);
maybeFreeI32(temp);
freeI64(r1);
@@ -5802,17 +6161,17 @@ BaseCompiler::endBlock(ExprType type)
// Save the value.
Maybe<AnyReg> r;
if (!deadCode_) {
r = popJoinRegUnlessVoid(type);
block.bceSafeOnExit &= bceSafe_;
}
// Leave the block.
- popStackOnBlockExit(block.framePushed);
+ fr.popStackOnBlockExit(block.stackHeight, deadCode_);
popValueStackTo(block.stackSize);
// Bind after cleanup: branches out will have popped the stack.
if (block.label.used()) {
masm.bind(&block.label);
// No value was provided by the fallthrough but the branch out will
// have stored one in joinReg, so capture that.
if (deadCode_)
@@ -5855,17 +6214,17 @@ BaseCompiler::endLoop(ExprType type)
Maybe<AnyReg> r;
if (!deadCode_) {
r = popJoinRegUnlessVoid(type);
// block.bceSafeOnExit need not be updated because it won't be used for
// the fallthrough path.
}
- popStackOnBlockExit(block.framePushed);
+ fr.popStackOnBlockExit(block.stackHeight, deadCode_);
popValueStackTo(block.stackSize);
// bceSafe_ stays the same along the fallthrough path because branches to
// loops branch to the top.
// Retain the value stored in joinReg by all paths.
if (!deadCode_)
pushJoinRegUnlessVoid(r);
@@ -5908,17 +6267,17 @@ BaseCompiler::emitIf()
return true;
}
void
BaseCompiler::endIfThen()
{
Control& ifThen = controlItem();
- popStackOnBlockExit(ifThen.framePushed);
+ fr.popStackOnBlockExit(ifThen.stackHeight, deadCode_);
popValueStackTo(ifThen.stackSize);
if (ifThen.otherLabel.used())
masm.bind(&ifThen.otherLabel);
if (ifThen.label.used())
masm.bind(&ifThen.label);
@@ -5946,17 +6305,17 @@ BaseCompiler::emitElse()
// Exit the "then" branch.
ifThenElse.deadThenBranch = deadCode_;
Maybe<AnyReg> r;
if (!deadCode_)
r = popJoinRegUnlessVoid(thenType);
- popStackOnBlockExit(ifThenElse.framePushed);
+ fr.popStackOnBlockExit(ifThenElse.stackHeight, deadCode_);
popValueStackTo(ifThenElse.stackSize);
if (!deadCode_)
masm.jump(&ifThenElse.label);
if (ifThenElse.otherLabel.used())
masm.bind(&ifThenElse.otherLabel);
@@ -5985,17 +6344,17 @@ BaseCompiler::endIfThenElse(ExprType typ
// we want to find there. The "then" arm has the same constraint.
Maybe<AnyReg> r;
if (!deadCode_) {
r = popJoinRegUnlessVoid(type);
ifThenElse.bceSafeOnExit &= bceSafe_;
}
- popStackOnBlockExit(ifThenElse.framePushed);
+ fr.popStackOnBlockExit(ifThenElse.stackHeight, deadCode_);
popValueStackTo(ifThenElse.stackSize);
if (ifThenElse.label.used())
masm.bind(&ifThenElse.label);
bool joinLive = !ifThenElse.deadOnArrival &&
(!ifThenElse.deadThenBranch || !deadCode_ || ifThenElse.label.bound());
@@ -6049,17 +6408,17 @@ BaseCompiler::emitBr()
Control& target = controlItem(relativeDepth);
target.bceSafeOnExit &= bceSafe_;
// Save any value in the designated join register, where the
// normal block exit code will also leave it.
Maybe<AnyReg> r = popJoinRegUnlessVoid(type);
- popStackBeforeBranch(target.framePushed);
+ fr.popStackBeforeBranch(target.stackHeight);
masm.jump(&target.label);
// The register holding the join value is free for the remainder
// of this block.
freeJoinRegUnlessVoid(r);
deadCode_ = true;
@@ -6079,17 +6438,17 @@ BaseCompiler::emitBrIf()
if (deadCode_) {
resetLatentOp();
return true;
}
Control& target = controlItem(relativeDepth);
target.bceSafeOnExit &= bceSafe_;
- BranchState b(&target.label, target.framePushed, InvertBranch(false), type);
+ BranchState b(&target.label, target.stackHeight, InvertBranch(false), type);
emitBranchSetup(&b);
emitBranchPerform(&b);
return true;
}
bool
BaseCompiler::emitBrTable()
@@ -6114,17 +6473,17 @@ BaseCompiler::emitBrTable()
Maybe<AnyReg> r = popJoinRegUnlessVoid(branchValueType);
Label dispatchCode;
masm.branch32(Assembler::Below, rc, Imm32(depths.length()), &dispatchCode);
// This is the out-of-range stub. rc is dead here but we don't need it.
- popStackBeforeBranch(controlItem(defaultDepth).framePushed);
+ fr.popStackBeforeBranch(controlItem(defaultDepth).stackHeight);
controlItem(defaultDepth).bceSafeOnExit &= bceSafe_;
masm.jump(&controlItem(defaultDepth).label);
// Emit stubs. rc is dead in all of these but we don't need it.
//
// The labels in the vector are in the TempAllocator and will
// be freed by and by.
//
@@ -6133,17 +6492,17 @@ BaseCompiler::emitBrTable()
LabelVector stubs;
if (!stubs.reserve(depths.length()))
return false;
for (uint32_t depth : depths) {
stubs.infallibleEmplaceBack(NonAssertingLabel());
masm.bind(&stubs.back());
- popStackBeforeBranch(controlItem(depth).framePushed);
+ fr.popStackBeforeBranch(controlItem(depth).stackHeight);
controlItem(depth).bceSafeOnExit &= bceSafe_;
masm.jump(&controlItem(depth).label);
}
// Emit table.
Label theTable;
jumpTable(stubs, &theTable);
@@ -6166,18 +6525,17 @@ bool
BaseCompiler::emitDrop()
{
if (!iter_.readDrop())
return false;
if (deadCode_)
return true;
- popStackIfMemory();
- popValueStackBy(1);
+ dropValue();
return true;
}
void
BaseCompiler::doReturn(ExprType type, bool popStack)
{
switch (type) {
case ExprType::Void: {
@@ -6602,47 +6960,47 @@ BaseCompiler::emitSetOrTeeLocal(uint32_t
if (deadCode_)
return true;
bceLocalIsUpdated(slot);
switch (locals_[slot]) {
case ValType::I32: {
RegI32 rv = popI32();
syncLocal(slot);
- storeToFrameI32(rv, frameOffsetFromSlot(slot, MIRType::Int32));
+ fr.storeLocalI32(rv, localFromSlot(slot, MIRType::Int32));
if (isSetLocal)
freeI32(rv);
else
pushI32(rv);
break;
}
case ValType::I64: {
RegI64 rv = popI64();
syncLocal(slot);
- storeToFrameI64(rv, frameOffsetFromSlot(slot, MIRType::Int64));
+ fr.storeLocalI64(rv, localFromSlot(slot, MIRType::Int64));
if (isSetLocal)
freeI64(rv);
else
pushI64(rv);
break;
}
case ValType::F64: {
RegF64 rv = popF64();
syncLocal(slot);
- storeToFrameF64(rv, frameOffsetFromSlot(slot, MIRType::Double));
+ fr.storeLocalF64(rv, localFromSlot(slot, MIRType::Double));
if (isSetLocal)
freeF64(rv);
else
pushF64(rv);
break;
}
case ValType::F32: {
RegF32 rv = popF32();
syncLocal(slot);
- storeToFrameF32(rv, frameOffsetFromSlot(slot, MIRType::Float32));
+ fr.storeLocalF32(rv, localFromSlot(slot, MIRType::Float32));
if (isSetLocal)
freeF32(rv);
else
pushF32(rv);
break;
}
default:
MOZ_CRASH("Local variable type");
@@ -7537,26 +7895,26 @@ BaseCompiler::emitAtomicRMW(ValType type
RegI32 tls = specific_edi;
RegI32 memoryBase = specific_edi; // Yes, same
masm.loadWasmTlsRegFromFrame(tls);
prepareMemoryAccess(&access, &check, tls, ptr);
masm.movl(Operand(Address(tls, offsetof(TlsData, memoryBase))), memoryBase);
- masm.Push(ecx);
- masm.Push(ebx);
+ fr.pushPtr(ecx);
+ fr.pushPtr(ebx);
RegI64 rd = specific_edx_eax;
BaseIndex srcAddr(memoryBase, ptr, TimesOne, access.offset());
Address value(esp, 0);
atomicRMW64(op, value, srcAddr, tmp, rd);
- masm.freeStack(8);
+ fr.popBytes(8);
pushI64(rd);
freeI32(specific_ecx);
freeI32(specific_edi);
freeI32(specific_esi);
#else // !JS_CODEGEN_X86
@@ -8491,146 +8849,41 @@ BaseCompiler::emitFunction()
return false;
if (!endFunction())
return false;
return true;
}
-void
-BaseCompiler::emitInitStackLocals()
-{
- MOZ_ASSERT(varLow_ < varHigh_, "there should be stack locals to initialize");
-
- static const uint32_t wordSize = sizeof(void*);
-
- // A local's localOffset always points above it in the frame, so that when
- // translated to a stack address we end up with an address pointing to the
- // base of the local. Thus to go from a raw frame offset to an SP offset we
- // first add K to the frame offset to obtain a localOffset for a slot of
- // size K, and then map that to an SP offset. Hence all the adjustments to
- // `low` in the offset calculations below.
-
- // On 64-bit systems we may have 32-bit alignment for the local area as it
- // may be preceded by parameters and prologue/debug data.
-
- uint32_t low = varLow_;
- if (low % wordSize) {
- masm.store32(Imm32(0), Address(StackPointer, localOffsetToSPOffset(low + 4)));
- low += 4;
- }
- MOZ_ASSERT(low % wordSize == 0);
-
- const uint32_t high = AlignBytes(varHigh_, wordSize);
- MOZ_ASSERT(high <= uint32_t(localSize_), "localSize_ should be aligned at least that");
-
- // An unrollLimit of 16 is chosen so that we only need an 8-bit signed
- // immediate to represent the offset in the store instructions in the loop
- // on x64.
-
- const uint32_t unrollLimit = 16;
- const uint32_t initWords = (high - low) / wordSize;
- const uint32_t tailWords = initWords % unrollLimit;
- const uint32_t loopHigh = high - (tailWords * wordSize);
-
- // With only one word to initialize, just store an immediate zero.
-
- if (initWords == 1) {
- masm.storePtr(ImmWord(0), Address(StackPointer, localOffsetToSPOffset(low + wordSize)));
- return;
- }
-
- // For other cases, it's best to have a zero in a register.
- //
- // One can do more here with SIMD registers (store 16 bytes at a time) or
- // with instructions like STRD on ARM (store 8 bytes at a time), but that's
- // for another day.
-
- RegI32 zero = needI32();
- masm.mov(ImmWord(0), zero);
-
- // For the general case we want to have a loop body of unrollLimit stores
- // and then a tail of less than unrollLimit stores. When initWords is less
- // than 2*unrollLimit the loop trip count is at most 1 and there is no
- // benefit to having the pointer calculations and the compare-and-branch.
- // So we completely unroll when we have initWords < 2 * unrollLimit. (In
- // this case we'll end up using 32-bit offsets on x64 for up to half of the
- // stores, though.)
-
- // Fully-unrolled case.
-
- if (initWords < 2 * unrollLimit) {
- for (uint32_t i = low; i < high; i += wordSize)
- masm.storePtr(zero, Address(StackPointer, localOffsetToSPOffset(i + wordSize)));
- freeI32(zero);
- return;
- }
-
- // Unrolled loop with a tail. Stores will use negative offsets. That's OK
- // for x86 and ARM, at least.
-
- // Compute pointer to the highest-addressed slot on the frame.
- RegI32 p = needI32();
- masm.computeEffectiveAddress(Address(StackPointer, localOffsetToSPOffset(low + wordSize)),
- p);
-
- // Compute pointer to the lowest-addressed slot on the frame that will be
- // initialized by the loop body.
- RegI32 lim = needI32();
- masm.computeEffectiveAddress(Address(StackPointer,
- localOffsetToSPOffset(loopHigh + wordSize)),
- lim);
-
- // The loop body. Eventually we'll have p == lim and exit the loop.
- Label again;
- masm.bind(&again);
- for (uint32_t i = 0; i < unrollLimit; ++i)
- masm.storePtr(zero, Address(p, -(wordSize * i)));
- masm.subPtr(Imm32(unrollLimit * wordSize), p);
- masm.branchPtr(Assembler::LessThan, lim, p, &again);
-
- // The tail.
- for (uint32_t i = 0; i < tailWords; ++i)
- masm.storePtr(zero, Address(p, -(wordSize * i)));
-
- freeI32(p);
- freeI32(lim);
- freeI32(zero);
-}
-
BaseCompiler::BaseCompiler(const ModuleEnvironment& env,
Decoder& decoder,
const FuncCompileInput& func,
const ValTypeVector& locals,
bool debugEnabled,
TempAllocator* alloc,
MacroAssembler* masm,
CompileMode mode)
: env_(env),
iter_(env, decoder),
func_(func),
lastReadCallSite_(0),
alloc_(*alloc),
locals_(locals),
- localSize_(0),
- varLow_(0),
- varHigh_(0),
- maxFramePushed_(0),
deadCode_(false),
debugEnabled_(debugEnabled),
bceSafe_(0),
- stackAddOffset_(0),
mode_(mode),
latentOp_(LatentOp::None),
latentType_(ValType::I32),
latentIntCmp_(Assembler::Equal),
latentDoubleCmp_(Assembler::DoubleEqual),
masm(*masm),
ra(*this),
+ fr(*masm),
#ifdef JS_CODEGEN_X64
specific_rax(RegI64(Register64(rax))),
specific_rcx(RegI64(Register64(rcx))),
specific_rdx(RegI64(Register64(rdx))),
#endif
#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
specific_eax(RegI32(eax)),
specific_ecx(RegI32(ecx)),
@@ -8675,39 +8928,19 @@ BaseCompiler::init()
return false;
}
if (!SigPILL_.append(MIRType::Pointer) || !SigPILL_.append(MIRType::Int32) ||
!SigPILL_.append(MIRType::Int64) || !SigPILL_.append(MIRType::Int64))
{
return false;
}
- if (!localInfo_.resize(locals_.length()))
+ if (!fr.setupLocals(localInfo_, locals_, sig().args(), debugEnabled_))
return false;
- const ValTypeVector& args = sig().args();
- BaseLocalIter i(locals_, args.length(), debugEnabled_);
- varLow_ = i.reservedSize();
- for (; !i.done() && i.index() < args.length(); i++) {
- MOZ_ASSERT(i.isArg());
- Local& l = localInfo_[i.index()];
- l.init(i.mirType(), i.frameOffset());
- varLow_ = i.currentLocalSize();
- }
-
- varHigh_ = varLow_;
- for (; !i.done() ; i++) {
- MOZ_ASSERT(!i.isArg());
- Local& l = localInfo_[i.index()];
- l.init(i.mirType(), i.frameOffset());
- varHigh_ = i.currentLocalSize();
- }
-
- localSize_ = AlignBytes(varHigh_, 16u);
-
addInterruptCheck();
return true;
}
FuncOffsets
BaseCompiler::finish()
{