Changeset 252800 in webkit

Timestamp:

Nov 22, 2019 2:13:28 PM (4 years ago)

Author:

Tadeu Zagallo

Message:

[WebAssembly] Improve Wasm::LLIntGenerator
​https://bugs.webkit.org/show_bug.cgi?id=204092

Reviewed by Saam Barati.

JSTests:

• wasm/stress/set-local-enclosed-stack.js: Added.

(assert.eq.instance.exports.foo):
(assert.eq):

Source/JavaScriptCore:

This improves the Wasm::LLIntGenerator by:

• Changing LLIntGenerator::ExpressionType from RefPtr<RegisterID> to VirtualRegister: Instead of allocating and retaining RegisterIDs we use VirtualRegisters directly and ensure that they match the WebAssembly stack, i.e. the parser's expression stack should match the virtual registers.
• Removing redundant moves when materializing constants and performing local.get: instead of creating a new temporary for each constant and local.get, we return the VirtualRegister for the constant/local slot directly. In order for this to work, we still allocate the stack slot for the temporaries, since we have to materialize them before loops and branches.
• Adding a constructor to ControlType that takes the results ExpressionList as an rvalue instead of copying it
• Optimizing callInformationFor, which is now split into two functions. The callee does not care about arguments, and should never allocate temporaries, and the caller case was optimized by avoiding unnecessary calls to newTemporary
• Delay holding the lock in LLintPlan::compileFunction, since we do not need to hold it while compiling the js-to-wasm entrypoint

• bytecode/BytecodeList.rb:
• bytecompiler/Label.h:

(JSC::GenericLabel::location const):
(JSC::GenericLabel::unresolvedJumps const):

• generator/Wasm.rb:
• llint/WebAssembly.asm:
• wasm/WasmAirIRGenerator.cpp:

(JSC::Wasm::AirIRGenerator::endTopLevel):
(JSC::Wasm::AirIRGenerator::didPopValueFromStack):

• wasm/WasmB3IRGenerator.cpp:

(JSC::Wasm::B3IRGenerator::endTopLevel):
(JSC::Wasm::B3IRGenerator::didPopValueFromStack):

• wasm/WasmFunctionCodeBlock.cpp:

(JSC::Wasm::FunctionCodeBlock::addJumpTable):

• wasm/WasmFunctionCodeBlock.h:
• wasm/WasmFunctionParser.h:

(JSC::Wasm::FunctionParser::expressionStack):
(JSC::Wasm::FunctionParser<Context>::parseBody):
(JSC::Wasm::FunctionParser<Context>::parseExpression):

• wasm/WasmLLIntGenerator.cpp:

(JSC::Wasm::LLIntGenerator::ControlType::loop):
(JSC::Wasm::LLIntGenerator::ControlType::topLevel):
(JSC::Wasm::LLIntGenerator::ControlType::block):
(JSC::Wasm::LLIntGenerator::ControlType::if_):
(JSC::Wasm::LLIntGenerator::ControlType::targetArity const):
(JSC::Wasm::LLIntGenerator::ControlType::stackSize const):
(JSC::Wasm::LLIntGenerator::ControlType::ControlType):
(JSC::Wasm::LLIntGenerator::unifyValuesWithBlock):
(JSC::Wasm::LLIntGenerator::push):
(JSC::Wasm::LLIntGenerator::didPopValueFromStack):
(JSC::Wasm::LLIntGenerator::emptyExpression):
(JSC::Wasm::LLIntGenerator::addEndToUnreachable):
(JSC::Wasm::LLIntGenerator::dump):
(JSC::Wasm::LLIntGenerator::virtualRegisterForWasmLocal):
(JSC::Wasm::LLIntGenerator::jsNullConstant):
(JSC::Wasm::LLIntGenerator::zeroConstant):
(JSC::Wasm::LLIntGenerator::getDropKeepCount):
(JSC::Wasm::LLIntGenerator::dropKeep):
(JSC::Wasm::LLIntGenerator::walkExpressionStack):
(JSC::Wasm::LLIntGenerator::checkConsistency):
(JSC::Wasm::LLIntGenerator::materializeConstantsAndLocals):
(JSC::Wasm::LLIntGenerator::materializeLocals):
(JSC::Wasm::LLIntGenerator::ConstantMapHashTraits::constructDeletedValue):
(JSC::Wasm::LLIntGenerator::ConstantMapHashTraits::isDeletedValue):
(JSC::Wasm::LLIntGenerator::LLIntGenerator):
(JSC::Wasm::LLIntGenerator::finalize):
(JSC::Wasm::LLIntGenerator::callInformationForCaller):
(JSC::Wasm::LLIntGenerator::callInformationForCallee):
(JSC::Wasm::LLIntGenerator::addArguments):
(JSC::Wasm::LLIntGenerator::addLocal):
(JSC::Wasm::LLIntGenerator::didFinishParsingLocals):
(JSC::Wasm::LLIntGenerator::addConstant):
(JSC::Wasm::LLIntGenerator::getLocal):
(JSC::Wasm::LLIntGenerator::setLocal):
(JSC::Wasm::LLIntGenerator::getGlobal):
(JSC::Wasm::LLIntGenerator::addLoop):
(JSC::Wasm::LLIntGenerator::addTopLevel):
(JSC::Wasm::LLIntGenerator::addBlock):
(JSC::Wasm::LLIntGenerator::addIf):
(JSC::Wasm::LLIntGenerator::addElse):
(JSC::Wasm::LLIntGenerator::addElseToUnreachable):
(JSC::Wasm::LLIntGenerator::addReturn):
(JSC::Wasm::LLIntGenerator::addBranch):
(JSC::Wasm::LLIntGenerator::addSwitch):
(JSC::Wasm::LLIntGenerator::endBlock):
(JSC::Wasm::LLIntGenerator::endTopLevel):
(JSC::Wasm::LLIntGenerator::addCall):
(JSC::Wasm::LLIntGenerator::addCallIndirect):
(JSC::Wasm::LLIntGenerator::addRefIsNull):
(JSC::Wasm::LLIntGenerator::addRefFunc):
(JSC::Wasm::LLIntGenerator::addTableGet):
(JSC::Wasm::LLIntGenerator::addTableSize):
(JSC::Wasm::LLIntGenerator::addTableGrow):
(JSC::Wasm::LLIntGenerator::addCurrentMemory):
(JSC::Wasm::LLIntGenerator::addGrowMemory):
(JSC::Wasm::LLIntGenerator::addSelect):
(JSC::Wasm::LLIntGenerator::load):
(JSC::GenericLabel<Wasm::GeneratorTraits>::setLocation):

• wasm/WasmLLIntPlan.cpp:

(JSC::Wasm::LLIntPlan::compileFunction):

• wasm/WasmValidate.cpp:

(JSC::Wasm::Validate::endTopLevel):
(JSC::Wasm::Validate::didPopValueFromStack):

Location:

trunk

Files:

• JSTests/ChangeLog (modified) (1 diff)
• JSTests/wasm/stress/set-local-enclosed-stack.js (added)
• Source/JavaScriptCore/ChangeLog (modified) (1 diff)
• Source/JavaScriptCore/bytecode/BytecodeList.rb (modified) (2 diffs)
• Source/JavaScriptCore/bytecompiler/Label.h (modified) (2 diffs)
• Source/JavaScriptCore/generator/Wasm.rb (modified) (2 diffs)
• Source/JavaScriptCore/llint/WebAssembly.asm (modified) (3 diffs)
• Source/JavaScriptCore/wasm/WasmAirIRGenerator.cpp (modified) (2 diffs)
• Source/JavaScriptCore/wasm/WasmB3IRGenerator.cpp (modified) (2 diffs)
• Source/JavaScriptCore/wasm/WasmFunctionCodeBlock.cpp (modified) (1 diff)
• Source/JavaScriptCore/wasm/WasmFunctionCodeBlock.h (modified) (1 diff)
• Source/JavaScriptCore/wasm/WasmFunctionParser.h (modified) (6 diffs)
• Source/JavaScriptCore/wasm/WasmLLIntGenerator.cpp (modified) (48 diffs)
• Source/JavaScriptCore/wasm/WasmLLIntPlan.cpp (modified) (1 diff)
• Source/JavaScriptCore/wasm/WasmValidate.cpp (modified) (2 diffs)

trunk/JSTests/ChangeLog

@@ +1 @@
+2019-11-22  Tadeu Zagallo  <tzagallo@apple.com>
+
+        [WebAssembly] Improve Wasm::LLIntGenerator
+        https://bugs.webkit.org/show_bug.cgi?id=204092
+
+        Reviewed by Saam Barati.
+
+        * wasm/stress/set-local-enclosed-stack.js: Added.
+        (assert.eq.instance.exports.foo):
+        (assert.eq):
+
 2019-11-22  Yusuke Suzuki  <ysuzuki@apple.com>
 

trunk/Source/JavaScriptCore/ChangeLog

@@ +1 @@
+2019-11-22  Tadeu Zagallo  <tzagallo@apple.com>
+
+        [WebAssembly] Improve Wasm::LLIntGenerator
+        https://bugs.webkit.org/show_bug.cgi?id=204092
+
+        Reviewed by Saam Barati.
+
+        This improves the Wasm::LLIntGenerator by:
+        - Changing LLIntGenerator::ExpressionType from RefPtr<RegisterID> to VirtualRegister: Instead of allocating and retaining
+          RegisterIDs we use VirtualRegisters directly and ensure that they match the WebAssembly stack, i.e. the parser's expression
+          stack should match the virtual registers.
+        - Removing redundant moves when materializing constants and performing local.get: instead of creating a new temporary
+          for each constant and local.get, we return the VirtualRegister for the constant/local slot directly. In order for this
+          to work, we still allocate the stack slot for the temporaries, since we have to materialize them before loops and branches.
+        - Adding a constructor to ControlType that takes the results ExpressionList as an rvalue instead of copying it
+        - Optimizing callInformationFor, which is now split into two functions. The callee does not care about arguments, and should
+          never allocate temporaries, and the caller case was optimized by avoiding unnecessary calls to newTemporary
+        - Delay holding the lock in LLintPlan::compileFunction, since we do not need to hold it while compiling the js-to-wasm entrypoint
+
+        * bytecode/BytecodeList.rb:
+        * bytecompiler/Label.h:
+        (JSC::GenericLabel::location const):
+        (JSC::GenericLabel::unresolvedJumps const):
+        * generator/Wasm.rb:
+        * llint/WebAssembly.asm:
+        * wasm/WasmAirIRGenerator.cpp:
+        (JSC::Wasm::AirIRGenerator::endTopLevel):
+        (JSC::Wasm::AirIRGenerator::didPopValueFromStack):
+        * wasm/WasmB3IRGenerator.cpp:
+        (JSC::Wasm::B3IRGenerator::endTopLevel):
+        (JSC::Wasm::B3IRGenerator::didPopValueFromStack):
+        * wasm/WasmFunctionCodeBlock.cpp:
+        (JSC::Wasm::FunctionCodeBlock::addJumpTable):
+        * wasm/WasmFunctionCodeBlock.h:
+        * wasm/WasmFunctionParser.h:
+        (JSC::Wasm::FunctionParser::expressionStack):
+        (JSC::Wasm::FunctionParser<Context>::parseBody):
+        (JSC::Wasm::FunctionParser<Context>::parseExpression):
+        * wasm/WasmLLIntGenerator.cpp:
+        (JSC::Wasm::LLIntGenerator::ControlType::loop):
+        (JSC::Wasm::LLIntGenerator::ControlType::topLevel):
+        (JSC::Wasm::LLIntGenerator::ControlType::block):
+        (JSC::Wasm::LLIntGenerator::ControlType::if_):
+        (JSC::Wasm::LLIntGenerator::ControlType::targetArity const):
+        (JSC::Wasm::LLIntGenerator::ControlType::stackSize const):
+        (JSC::Wasm::LLIntGenerator::ControlType::ControlType):
+        (JSC::Wasm::LLIntGenerator::unifyValuesWithBlock):
+        (JSC::Wasm::LLIntGenerator::push):
+        (JSC::Wasm::LLIntGenerator::didPopValueFromStack):
+        (JSC::Wasm::LLIntGenerator::emptyExpression):
+        (JSC::Wasm::LLIntGenerator::addEndToUnreachable):
+        (JSC::Wasm::LLIntGenerator::dump):
+        (JSC::Wasm::LLIntGenerator::virtualRegisterForWasmLocal):
+        (JSC::Wasm::LLIntGenerator::jsNullConstant):
+        (JSC::Wasm::LLIntGenerator::zeroConstant):
+        (JSC::Wasm::LLIntGenerator::getDropKeepCount):
+        (JSC::Wasm::LLIntGenerator::dropKeep):
+        (JSC::Wasm::LLIntGenerator::walkExpressionStack):
+        (JSC::Wasm::LLIntGenerator::checkConsistency):
+        (JSC::Wasm::LLIntGenerator::materializeConstantsAndLocals):
+        (JSC::Wasm::LLIntGenerator::materializeLocals):
+        (JSC::Wasm::LLIntGenerator::ConstantMapHashTraits::constructDeletedValue):
+        (JSC::Wasm::LLIntGenerator::ConstantMapHashTraits::isDeletedValue):
+        (JSC::Wasm::LLIntGenerator::LLIntGenerator):
+        (JSC::Wasm::LLIntGenerator::finalize):
+        (JSC::Wasm::LLIntGenerator::callInformationForCaller):
+        (JSC::Wasm::LLIntGenerator::callInformationForCallee):
+        (JSC::Wasm::LLIntGenerator::addArguments):
+        (JSC::Wasm::LLIntGenerator::addLocal):
+        (JSC::Wasm::LLIntGenerator::didFinishParsingLocals):
+        (JSC::Wasm::LLIntGenerator::addConstant):
+        (JSC::Wasm::LLIntGenerator::getLocal):
+        (JSC::Wasm::LLIntGenerator::setLocal):
+        (JSC::Wasm::LLIntGenerator::getGlobal):
+        (JSC::Wasm::LLIntGenerator::addLoop):
+        (JSC::Wasm::LLIntGenerator::addTopLevel):
+        (JSC::Wasm::LLIntGenerator::addBlock):
+        (JSC::Wasm::LLIntGenerator::addIf):
+        (JSC::Wasm::LLIntGenerator::addElse):
+        (JSC::Wasm::LLIntGenerator::addElseToUnreachable):
+        (JSC::Wasm::LLIntGenerator::addReturn):
+        (JSC::Wasm::LLIntGenerator::addBranch):
+        (JSC::Wasm::LLIntGenerator::addSwitch):
+        (JSC::Wasm::LLIntGenerator::endBlock):
+        (JSC::Wasm::LLIntGenerator::endTopLevel):
+        (JSC::Wasm::LLIntGenerator::addCall):
+        (JSC::Wasm::LLIntGenerator::addCallIndirect):
+        (JSC::Wasm::LLIntGenerator::addRefIsNull):
+        (JSC::Wasm::LLIntGenerator::addRefFunc):
+        (JSC::Wasm::LLIntGenerator::addTableGet):
+        (JSC::Wasm::LLIntGenerator::addTableSize):
+        (JSC::Wasm::LLIntGenerator::addTableGrow):
+        (JSC::Wasm::LLIntGenerator::addCurrentMemory):
+        (JSC::Wasm::LLIntGenerator::addGrowMemory):
+        (JSC::Wasm::LLIntGenerator::addSelect):
+        (JSC::Wasm::LLIntGenerator::load):
+        (JSC::GenericLabel<Wasm::GeneratorTraits>::setLocation):
+        * wasm/WasmLLIntPlan.cpp:
+        (JSC::Wasm::LLIntPlan::compileFunction):
+        * wasm/WasmValidate.cpp:
+        (JSC::Wasm::Validate::endTopLevel):
+        (JSC::Wasm::Validate::didPopValueFromStack):
+
 2019-11-22  Yusuke Suzuki  <ysuzuki@apple.com>
 

trunk/Source/JavaScriptCore/bytecode/BytecodeList.rb

@@ -1300 +1300 @@
         scrutinee: VirtualRegister,
         tableIndex: unsigned,
-        defaultTarget: WasmBoundLabel,
     }
 
@@ -1307 +1306 @@
 op :unreachable
 op :ret_void
+
+op :drop_keep,
+    args: {
+        startOffset: unsigned,
+        dropCount: unsigned,
+        keepCount: unsigned,
+    }
 
 op :ref_is_null,

trunk/Source/JavaScriptCore/bytecompiler/Label.h

@@ -129 +129 @@
         using BytecodeGenerator = BytecodeGeneratorBase<Traits>;
         using BoundLabel = GenericBoundLabel<Traits>;
+        using JumpVector = Vector<int, 8>;
 
     public:
@@ -171 +172 @@
         bool isBound() const { return m_bound; }
 
+        unsigned location() const
+        {
+            ASSERT(!isForward());
+            m_bound = true;
+            return m_location;
+        };
+
+        const JumpVector& unresolvedJumps() const { return  m_unresolvedJumps; }
+
     private:
         friend BoundLabel;
-
-        typedef Vector<int, 8> JumpVector;
 
         static constexpr unsigned invalidLocation = UINT_MAX;

trunk/Source/JavaScriptCore/generator/Wasm.rb

@@ -59 +59 @@
 auto LLIntGenerator::addOp<#{op_type(op)}>(ExpressionType lhs, ExpressionType rhs, ExpressionType& result) -> PartialResult
 {
-    result = lhs;
+    result = push();
     #{op.capitalized_name}::emit(this, result, lhs, rhs);
     return { };
@@ -71 +71 @@
 auto LLIntGenerator::addOp<#{op_type(op)}>(ExpressionType operand, ExpressionType& result) -> PartialResult
 {
-    result = operand;
+    result = push();
     #{op.capitalized_name}::emit(this, result, operand);
     return { };

trunk/Source/JavaScriptCore/llint/WebAssembly.asm

@@ -298 +298 @@
     negi ws1
     sxi2q ws1, ws1
-    leap (NumberOfWasmArguments + CalleeSaveSpaceAsVirtualRegisters) * -8[cfr], ws0
+    leap (NumberOfWasmArguments + CalleeSaveSpaceAsVirtualRegisters + 1) * -8[cfr], ws0
 .zeroInitializeLocalsLoop:
     addq 1, ws1
@@ -589 +589 @@
 
     loadi VectorSizeOffset[t2], t3
-    biaeq t0, t3, .default
+    bib t0, t3, .inBounds
+
+.outOfBounds:
+    subi t3, 1, t0
+
+.inBounds:
     loadp VectorBufferOffset[t2], t2
-    loadi [t2, t0, 4], t3
+    muli sizeof Wasm::FunctionCodeBlock::JumpTableEntry, t0
+
+    loadi Wasm::FunctionCodeBlock::JumpTableEntry::startOffset[t2, t0], t1
+    loadi Wasm::FunctionCodeBlock::JumpTableEntry::dropCount[t2, t0], t3
+    loadi Wasm::FunctionCodeBlock::JumpTableEntry::keepCount[t2, t0], t5
+    dropKeep(t1, t3, t5)
+
+    loadis Wasm::FunctionCodeBlock::JumpTableEntry::target[t2, t0], t3
     assert(macro(ok) btinz t3, .ok end)
     wasmDispatchIndirect(t3)
-
-.default:
-    jump(ctx, m_defaultTarget)
 end)
 
@@ -1994 +2003 @@
     returnq(ctx, t0)
 end)
+
+macro dropKeep(startOffset, drop, keep)
+    lshifti 3, startOffset
+    subp cfr, startOffset, startOffset
+    negi drop
+    sxi2q drop, drop
+
+.copyLoop:
+    btiz keep, .done
+    loadq [startOffset, drop, 8], t6
+    storeq t6, [startOffset]
+    subi 1, keep
+    subp 8, startOffset
+    jmp .copyLoop
+
+.done:
+end
+
+wasmOp(drop_keep, WasmDropKeep, macro(ctx)
+    wgetu(ctx, m_startOffset, t0)
+    wgetu(ctx, m_dropCount, t1)
+    wgetu(ctx, m_keepCount, t2)
+
+    dropKeep(t0, t1, t2)
+
+    dispatch(ctx)
+end)

trunk/Source/JavaScriptCore/wasm/WasmAirIRGenerator.cpp

@@ -286 +286 @@
     PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&, const Stack& expressionStack = { });
 
+    PartialResult WARN_UNUSED_RETURN endTopLevel(BlockSignature, const Stack&) { return { }; }
+
     // Calls
     PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, Vector<ExpressionType, 1>& results);
@@ -300 +302 @@
     void setParser(FunctionParser<AirIRGenerator>* parser) { m_parser = parser; };
     void didFinishParsingLocals() { }
+    void didPopValueFromStack() { }
 
     static Vector<Tmp> toTmpVector(const Vector<TypedTmp>& vector)

trunk/Source/JavaScriptCore/wasm/WasmB3IRGenerator.cpp

@@ -236 +236 @@
     PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&, const Stack& = { });
 
+    PartialResult WARN_UNUSED_RETURN endTopLevel(BlockSignature, const Stack&) { return { }; }
+
     // Calls
     PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ResultList& results);
@@ -245 +247 @@
     void setParser(FunctionParser<B3IRGenerator>* parser) { m_parser = parser; };
     void didFinishParsingLocals() { }
+    void didPopValueFromStack() { }
 
     Value* constant(B3::Type, uint64_t bits, Optional<Origin> = WTF::nullopt);

trunk/Source/JavaScriptCore/wasm/WasmFunctionCodeBlock.cpp

@@ -76 +76 @@
 auto FunctionCodeBlock::addJumpTable(size_t numberOfEntries) -> JumpTable&
 {
-#if !ASSERT_DISABLED
-    m_jumpTables.append(JumpTable(numberOfEntries, 0));
-#else
     m_jumpTables.append(JumpTable(numberOfEntries));
-#endif
     return m_jumpTables.last();
 }

trunk/Source/JavaScriptCore/wasm/WasmFunctionCodeBlock.h

@@ -106 +106 @@
     const Signature& signature(unsigned index) const;
 
-    using JumpTable = Vector<InstructionStream::Offset>;
+    struct JumpTableEntry {
+        int target { 0 };
+        unsigned startOffset;
+        unsigned dropCount;
+        unsigned keepCount;
+    };
+
+    using JumpTable = Vector<JumpTableEntry>;
     JumpTable& addJumpTable(size_t numberOfEntries);
     const JumpTable& jumpTable(unsigned tableIndex) const;

trunk/Source/JavaScriptCore/wasm/WasmFunctionParser.h

@@ -78 +78 @@
 
     Vector<ControlEntry>& controlStack() { return m_controlStack; }
+    Stack& expressionStack() { return m_expressionStack; }
 
 private:
@@ -90 +91 @@
         WASM_PARSER_FAIL_IF(m_expressionStack.isEmpty(), "can't pop empty stack in " what); \
         result = m_expressionStack.takeLast();                                              \
+        m_context.didPopValueFromStack();                                                   \
     } while (0)
 
@@ -177 +179 @@
         }
     }
+    WASM_FAIL_IF_HELPER_FAILS(m_context.endTopLevel(&m_signature, m_expressionStack));
 
     ASSERT(op == OpType::End);
@@ -439 +442 @@
         Vector<ExpressionType> args;
         WASM_PARSER_FAIL_IF(!args.tryReserveCapacity(calleeSignature.argumentCount()), "can't allocate enough memory for call's ", calleeSignature.argumentCount(), " arguments");
-        for (size_t i = firstArgumentIndex; i < m_expressionStack.size(); ++i)
+        for (size_t i = firstArgumentIndex; i < m_expressionStack.size(); ++i) {
             args.uncheckedAppend(m_expressionStack.at(i));
+            m_context.didPopValueFromStack();
+        }
         m_expressionStack.shrink(firstArgumentIndex);
 
@@ -468 +473 @@
         WASM_PARSER_FAIL_IF(!args.tryReserveCapacity(argumentCount), "can't allocate enough memory for ", argumentCount, " call_indirect arguments");
         size_t firstArgumentIndex = m_expressionStack.size() - argumentCount;
-        for (size_t i = firstArgumentIndex; i < m_expressionStack.size(); ++i)
+        for (size_t i = firstArgumentIndex; i < m_expressionStack.size(); ++i) {
             args.uncheckedAppend(m_expressionStack.at(i));
+            m_context.didPopValueFromStack();
+        }
         m_expressionStack.shrink(firstArgumentIndex);
 
@@ -609 +616 @@
         WASM_PARSER_FAIL_IF(!m_expressionStack.size(), "can't drop on empty stack");
         m_expressionStack.takeLast();
+        m_context.didPopValueFromStack();
         return { };
     }

trunk/Source/JavaScriptCore/wasm/WasmLLIntGenerator.cpp

@@ -33 +33 @@
 #include "InstructionStream.h"
 #include "Label.h"
-#include "RegisterID.h"
 #include "WasmCallingConvention.h"
 #include "WasmContextInlines.h"
@@ -40 +39 @@
 #include "WasmGeneratorTraits.h"
 #include "WasmThunks.h"
+#include <wtf/CompletionHandler.h>
 #include <wtf/RefPtr.h>
 #include <wtf/StdUnorderedMap.h>
@@ -48 +48 @@
 class LLIntGenerator : public BytecodeGeneratorBase<GeneratorTraits> {
 public:
-    using ExpressionType = RefPtr<RegisterID>;
+    using ExpressionType = VirtualRegister;
     using ExpressionList = Vector<ExpressionType, 1>;
-    using Stack = ExpressionList;
+    using Stack = Vector<ExpressionType, 16, UnsafeVectorOverflow>;
 
     struct ControlLoop  {
@@ -74 +74 @@
         }
 
-        static ControlType loop(BlockSignature signature, const ExpressionList& results, Ref<Label> body, RefPtr<Label> continuation)
+        static ControlType loop(BlockSignature signature, unsigned stackSize, Ref<Label>&& body, RefPtr<Label>&& continuation)
         {
-            return ControlType(signature, results, WTFMove(continuation), ControlLoop { WTFMove(body) });
-        }
-
-        static ControlType topLevel(BlockSignature signature, const ExpressionList& results, RefPtr<Label> continuation)
+            return ControlType(signature, stackSize - signature->argumentCount(), WTFMove(continuation), ControlLoop { WTFMove(body) });
+        }
+
+        static ControlType topLevel(BlockSignature signature, unsigned stackSize, RefPtr<Label>&& continuation)
         {
-            return ControlType(signature, results, WTFMove(continuation), ControlTopLevel { });
-        }
-
-        static ControlType block(BlockSignature signature, const ExpressionList& results, RefPtr<Label> continuation)
+            return ControlType(signature, stackSize, WTFMove(continuation), ControlTopLevel { });
+        }
+
+        static ControlType block(BlockSignature signature, unsigned stackSize, RefPtr<Label>&& continuation)
         {
-            return ControlType(signature, results, WTFMove(continuation), ControlBlock { });
-        }
-
-        static ControlType if_(BlockSignature signature, const ExpressionList& results, Ref<Label> alternate, RefPtr<Label> continuation)
+            return ControlType(signature, stackSize - signature->argumentCount(), WTFMove(continuation), ControlBlock { });
+        }
+
+        static ControlType if_(BlockSignature signature, unsigned stackSize, Ref<Label>&& alternate, RefPtr<Label>&& continuation)
         {
-            return ControlType(signature, results, WTFMove(continuation), ControlIf { WTFMove(alternate) });
+            return ControlType(signature, stackSize - signature->argumentCount(), WTFMove(continuation), ControlIf { WTFMove(alternate) });
         }
 
@@ -101 +101 @@
         }
 
+        unsigned targetArity() const
+        {
+            if (WTF::holds_alternative<ControlLoop>(*this))
+                return m_signature->argumentCount();
+            return m_signature->returnCount();
+        }
+
+        unsigned stackSize() const { return m_stackSize; }
+
         BlockSignature m_signature;
-        ExpressionList m_results;
+        unsigned m_stackSize;
         RefPtr<Label> m_continuation;
 
     private:
         template<typename T>
-        ControlType(BlockSignature signature, const ExpressionList& results, RefPtr<Label> continuation, T t)
+        ControlType(BlockSignature signature, unsigned stackSize, RefPtr<Label>&& continuation, T&& t)
             : Base(WTFMove(t))
             , m_signature(signature)
-            , m_results(results)
+            , m_stackSize(stackSize)
             , m_continuation(WTFMove(continuation))
         {
@@ -138 +147 @@
     {
         ASSERT(destinations.size() <= values.size());
+        auto offset = values.size() - destinations.size();
         for (size_t i = 0; i < destinations.size(); ++i)
-            WasmMov::emit(this, destinations[destinations.size() - i - 1], values[values.size() - i - 1]);
-    }
-
-
-    static ExpressionType emptyExpression() { return nullptr; };
+            WasmMov::emit(this, destinations[i], values[offset + i]);
+    }
+
+    enum NoConsistencyCheckTag { NoConsistencyCheck };
+    ExpressionType push(NoConsistencyCheckTag)
+    {
+        m_maxStackSize = std::max(m_maxStackSize, ++m_stackSize);
+        return virtualRegisterForLocal(m_stackSize - 1);
+    }
+
+    ExpressionType push()
+    {
+        checkConsistency();
+        return push(NoConsistencyCheck);
+    }
+
+    void didPopValueFromStack() { --m_stackSize; }
+
+    static ExpressionType emptyExpression() { return VirtualRegister { }; };
     Stack createStack() { return Stack(); }
     bool isControlTypeIf(const ControlType& control) { return WTF::holds_alternative<ControlIf>(control); }
@@ -149 +173 @@
     PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
     PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
-    ExpressionType addConstant(Type, uint64_t);
+    ExpressionType addConstant(Type, int64_t);
 
     // References
@@ -188 +212 @@
     PartialResult WARN_UNUSED_RETURN addLoop(BlockSignature, Stack& enclosingStack, ControlType& block, Stack& newStack, uint32_t loopIndex);
     PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, BlockSignature, Stack& enclosingStack, ControlType& result, Stack& newStack);
-    PartialResult WARN_UNUSED_RETURN addElse(ControlType&, const ExpressionList&);
+    PartialResult WARN_UNUSED_RETURN addElse(ControlType&, Stack&);
     PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlType&);
 
-    PartialResult WARN_UNUSED_RETURN addReturn(const ControlType&, const ExpressionList& returnValues);
-    PartialResult WARN_UNUSED_RETURN addBranch(ControlType&, ExpressionType condition, const ExpressionList& returnValues);
-    PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlType*>& targets, ControlType& defaultTargets, const ExpressionList& expressionStack);
-    PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
-    PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&, const Stack& expressionStack = { });
+    PartialResult WARN_UNUSED_RETURN addReturn(const ControlType&, Stack& returnValues);
+    PartialResult WARN_UNUSED_RETURN addBranch(ControlType&, ExpressionType condition, Stack& returnValues);
+    PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlType*>& targets, ControlType& defaultTargets, Stack& expressionStack);
+    PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, Stack& expressionStack);
+    PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&, const Stack& expressionStack = { }, bool unreachable = true);
+    PartialResult WARN_UNUSED_RETURN endTopLevel(BlockSignature, const Stack&);
 
     // Calls
@@ -206 +231 @@
     void setParser(FunctionParser<LLIntGenerator>* parser) { m_parser = parser; };
 
-    void dump(const Vector<ControlEntry>&, const ExpressionList*) { }
+    void dump(const Vector<ControlEntry>&, const Stack*) { }
 
 private:
@@ -215 +240 @@
         unsigned numberOfStackArguments;
         ExpressionList arguments;
-        ExpressionList results;
+        CompletionHandler<void(ExpressionList&)> commitResults;
     };
 
-    LLIntCallInformation callInformationFor(const Signature&, CallRole = CallRole::Caller);
-
-    VirtualRegister virtualRegisterForLocal(uint32_t index)
+    LLIntCallInformation callInformationForCaller(const Signature&);
+    Vector<VirtualRegister, 2> callInformationForCallee(const Signature&);
+
+    VirtualRegister virtualRegisterForWasmLocal(uint32_t index)
     {
         if (index < m_codeBlock->m_numArguments)
@@ -228 +254 @@
         const uint32_t gprCount = callingConvention.gprArgs.size();
         const uint32_t fprCount = callingConvention.fprArgs.size();
-        return ::JSC::virtualRegisterForLocal(index - m_codeBlock->m_numArguments + gprCount + fprCount + numberOfLLIntCalleeSaveRegisters);
-    }
-
-    ExpressionList tmpsForSignature(BlockSignature signature)
-    {
-        ExpressionList result(signature->returnCount());
-        for (unsigned i = 0; i < signature->returnCount(); ++i)
-            result[i] = newTemporary();
+        return virtualRegisterForLocal(index - m_codeBlock->m_numArguments + gprCount + fprCount + numberOfLLIntCalleeSaveRegisters);
+    }
+
+    ExpressionType jsNullConstant()
+    {
+        if (UNLIKELY(!m_jsNullConstant.isValid())) {
+            m_jsNullConstant = VirtualRegister(FirstConstantRegisterIndex + m_codeBlock->m_constants.size());
+            m_codeBlock->m_constants.append(JSValue::encode(jsNull()));
+            if (UNLIKELY(Options::dumpGeneratedWasmBytecodes()))
+                m_codeBlock->m_constantTypes.append(Type::Anyref);
+        }
+        return m_jsNullConstant;
+    }
+
+    ExpressionType zeroConstant()
+    {
+        if (UNLIKELY(!m_zeroConstant.isValid())) {
+            m_zeroConstant = VirtualRegister(FirstConstantRegisterIndex + m_codeBlock->m_constants.size());
+            m_codeBlock->m_constants.append(0);
+            if (UNLIKELY(Options::dumpGeneratedWasmBytecodes()))
+                m_codeBlock->m_constantTypes.append(Type::I32);
+        }
+        return m_zeroConstant;
+    }
+
+    void getDropKeepCount(const ControlType& target, unsigned& startOffset, unsigned& drop, unsigned& keep)
+    {
+        startOffset = target.stackSize() + 1;
+        keep = target.targetArity();
+        drop = m_stackSize - target.stackSize() - target.targetArity();
+    }
+
+    void dropKeep(Stack& values, const ControlType& target, bool dropValues)
+    {
+        unsigned startOffset;
+        unsigned keep;
+        unsigned drop;
+
+        getDropKeepCount(target, startOffset, drop, keep);
+
+        if (dropValues)
+            values.shrink(keep);
+
+        if (!drop)
+            return;
+
+        if (keep)
+            WasmDropKeep::emit(this, startOffset, drop, keep);
+    }
+
+    template<typename Functor>
+    void walkExpressionStack(Stack& expressionStack, unsigned stackSize, const Functor& functor)
+    {
+        for (unsigned i = expressionStack.size(); i > 0; --i) {
+            VirtualRegister slot = virtualRegisterForLocal(stackSize - i);
+            functor(expressionStack[expressionStack.size() - i], slot);
+        }
+    }
+
+    template<typename Functor>
+    void walkExpressionStack(Stack& expressionStack, const Functor& functor)
+    {
+        walkExpressionStack(expressionStack, m_stackSize, functor);
+    }
+
+    template<typename Functor>
+    void walkExpressionStack(ControlEntry& entry, const Functor& functor)
+    {
+        walkExpressionStack(entry.enclosedExpressionStack, entry.controlData.stackSize(), functor);
+    }
+
+    void checkConsistency()
+    {
+#if !ASSERT_DISABLED
+        // The rules for locals and constants in the stack are:
+        // 1) Locals have to be materialized whenever a control entry is pushed to the control stack (i.e. every time we splitStack)
+        //    NOTE: This is a trade-off so that set_local does not have to walk up the control stack looking for delayed get_locals
+        // 2) If the control entry is a loop, we also need to materialize constants in the newStack, since those slots will be written
+        //    to from loop back edges
+        // 3) Both locals and constants have to be materialized before branches, since multiple branches might share the same target,
+        //    we can't make any assumptions about the stack state at that point, so we materialize the stack.
+        for (ControlEntry& controlEntry : m_parser->controlStack()) {
+            walkExpressionStack(controlEntry, [&](VirtualRegister expression, VirtualRegister slot) {
+                ASSERT(expression == slot || expression.isConstant());
+            });
+        }
+        walkExpressionStack(m_parser->expressionStack(), [&](VirtualRegister expression, VirtualRegister slot) {
+            ASSERT(expression == slot || expression.isConstant() || expression.isArgument() || expression.toLocal() < m_codeBlock->m_numVars);
+        });
+#endif
+    }
+
+    void materializeConstantsAndLocals(Stack& expressionStack)
+    {
+        if (expressionStack.isEmpty())
+            return;
+
+        checkConsistency();
+        walkExpressionStack(expressionStack, [&](VirtualRegister& expression, VirtualRegister slot) {
+            ASSERT(expression == slot || expression.isConstant() || expression.isArgument() || expression.toLocal() < m_codeBlock->m_numVars);
+            if (expression == slot)
+                return;
+            WasmMov::emit(this, slot, expression);
+            expression = slot;
+        });
+        checkConsistency();
+    }
+
+    Stack splitStack(BlockSignature signature, Stack& stack)
+    {
+        Stack result = JSC::Wasm::splitStack(signature, stack);
+
+        m_stackSize -= result.size();
+        checkConsistency();
+        walkExpressionStack(stack, [&](VirtualRegister& expression, VirtualRegister slot) {
+            ASSERT(expression == slot || expression.isConstant() || expression.isArgument() || expression.toLocal() < m_codeBlock->m_numVars);
+            if (expression == slot || expression.isConstant())
+                return;
+            WasmMov::emit(this, slot, expression);
+            expression = slot;
+        });
+        checkConsistency();
+        m_stackSize += result.size();
         return result;
-    }
-
-    ExpressionType jsNullConstant()
-    {
-        if (!m_jsNullConstant)
-            m_jsNullConstant = addConstant(Type::Anyref, JSValue::encode(jsNull()));
-        return m_jsNullConstant;
     }
 
     struct SwitchEntry {
         InstructionStream::Offset offset;
-        InstructionStream::Offset* jumpTarget;
+        int* jumpTarget;
+    };
+
+    struct ConstantMapHashTraits : WTF::GenericHashTraits<EncodedJSValue> {
+        static constexpr bool emptyValueIsZero = true;
+        static void constructDeletedValue(EncodedJSValue& slot) { slot = JSValue::encode(jsNull()); }
+        static bool isDeletedValue(EncodedJSValue value) { return value == JSValue::encode(jsNull()); }
     };
 
@@ -257 +397 @@
     HashMap<Label*, Vector<SwitchEntry>> m_switches;
     ExpressionType m_jsNullConstant;
+    ExpressionType m_zeroConstant;
     ExpressionList m_unitializedLocals;
-    StdUnorderedMap<uint64_t, VirtualRegister> m_constantMap;
+    HashMap<EncodedJSValue, VirtualRegister, WTF::IntHash<EncodedJSValue>, ConstantMapHashTraits> m_constantMap;
+    Vector<VirtualRegister, 2> m_results;
+    unsigned m_stackSize { 0 };
+    unsigned m_maxStackSize { 0 };
 };
 
@@ -271 +415 @@
 
 LLIntGenerator::LLIntGenerator(const ModuleInformation& info, unsigned functionIndex, ThrowWasmException throwWasmException, const Signature&)
-    : BytecodeGeneratorBase(makeUnique<FunctionCodeBlock>(functionIndex), numberOfLLIntCalleeSaveRegisters)
+    : BytecodeGeneratorBase(makeUnique<FunctionCodeBlock>(functionIndex), 0)
     , m_info(info)
     , m_functionIndex(functionIndex)
 {
+    m_codeBlock->m_numVars = numberOfLLIntCalleeSaveRegisters;
+    m_stackSize = numberOfLLIntCalleeSaveRegisters;
+    m_maxStackSize = numberOfLLIntCalleeSaveRegisters;
+
     if (throwWasmException)
         Thunks::singleton().setThrowWasmException(throwWasmException);
@@ -284 +432 @@
 {
     RELEASE_ASSERT(m_codeBlock);
+    m_codeBlock->m_numCalleeLocals = WTF::roundUpToMultipleOf(stackAlignmentRegisters(), m_maxStackSize);
     m_codeBlock->setInstructions(m_writer.finalize());
     return WTFMove(m_codeBlock);
@@ -291 +440 @@
 #include "WasmLLIntGeneratorInlines.h"
 
-auto LLIntGenerator::callInformationFor(const Signature& signature, CallRole role) -> LLIntCallInformation
-{
+auto LLIntGenerator::callInformationForCaller(const Signature& signature) -> LLIntCallInformation
+{
+    // This function sets up the stack layout for calls. The desired stack layout is:
+
+    // FPRn
+    // ...
+    // FPR1
+    // FPR0
+    // ---
+    // GPRn
+    // ...
+    // GPR1
+    // GPR0
+    // ----
+    // stackN
+    // ...
+    // stack1
+    // stack0
+    // ---
+    // call frame header
+
+    // We need to allocate at least space for all GPRs and FPRs.
+    // Return values use the same allocation layout.
+
+    const auto initialStackSize = m_stackSize;
+
     const auto& callingConvention = wasmCallingConvention();
     const uint32_t gprCount = callingConvention.gprArgs.size();
@@ -300 +473 @@
     uint32_t gprIndex = 0;
     uint32_t fprIndex = 0;
-
-    Vector<RefPtr<RegisterID>, 16> registers;
-
-    ExpressionList arguments(signature.argumentCount());
-    ExpressionList results(signature.returnCount());
+    uint32_t stackIndex = 0;
 
     auto allocateStackRegister = [&](Type type) {
-        ASSERT(role == CallRole::Caller);
         switch (type) {
         case Type::I32:
@@ -315 +483 @@
             if (gprIndex < gprCount)
                 ++gprIndex;
-            else {
-                registers.append(newTemporary());
+            else if (stackIndex++ >= stackCount)
                 ++stackCount;
-            }
             break;
         case Type::F32:
@@ -324 +490 @@
             if (fprIndex < fprCount)
                 ++fprIndex;
-            else {
-                registers.append(newTemporary());
+            else if (stackIndex++ >= stackCount)
                 ++stackCount;
-            }
             break;
         case Void:
@@ -336 +500 @@
 
 
-    if (role == CallRole::Callee) {
-        // Reuse the slots we allocated to spill the registers in addArguments
-        for (uint32_t i = gprCount + fprCount; i--;)
-            registers.append(new RegisterID(::JSC::virtualRegisterForLocal(numberOfLLIntCalleeSaveRegisters + i)));
-    } else {
-        for (uint32_t i = 0; i < gprCount; i++)
-            registers.append(newTemporary());
-        for (uint32_t i = 0; i < fprCount; i++)
-            registers.append(newTemporary());
-
-        for (uint32_t i = 0; i < signature.argumentCount(); i++)
-            allocateStackRegister(signature.argument(i));
-        gprIndex = 0;
-        fprIndex = 0;
-        for (uint32_t i = 0; i < signature.returnCount(); i++)
-            allocateStackRegister(signature.returnType(i));
-    }
-
-    unsigned stackOffset;
-    if (role == CallRole::Callee)
-        stackOffset = static_cast<unsigned>(-registers.last()->index());
-    else {
-        // Align the stack
-        auto computeStackOffset = [&] {
-            return static_cast<unsigned>(-registers.last()->index()) + CallFrame::headerSizeInRegisters;
-        };
-        while (computeStackOffset() % stackAlignmentRegisters())
-            registers.append(newTemporary());
-        stackOffset = computeStackOffset();
-    }
-
-    ASSERT(role == CallRole::Caller || !stackCount);
-    const uint32_t maxGPRIndex = stackCount + gprCount;
+    for (uint32_t i = 0; i < signature.argumentCount(); i++)
+        allocateStackRegister(signature.argument(i));
+
+    gprIndex = 0;
+    fprIndex = 0;
+    stackIndex = 0;
+    for (uint32_t i = 0; i < signature.returnCount(); i++)
+        allocateStackRegister(signature.returnType(i));
+
+    // FIXME: we are allocating the extra space for the argument/return count in order to avoid interference, but we could do better
+    // NOTE: We increase arg count by 1 for the case of indirect calls
+    m_stackSize += std::max(signature.argumentCount() + 1, signature.returnCount()) + gprCount + fprCount + stackCount + CallFrame::headerSizeInRegisters;
+    if (m_stackSize % stackAlignmentRegisters())
+        ++m_stackSize;
+    if (m_maxStackSize < m_stackSize)
+        m_maxStackSize = m_stackSize;
+
+
+    ExpressionList arguments(signature.argumentCount());
+    ExpressionList temporaryResults(signature.returnCount());
+
+    const unsigned stackOffset = m_stackSize;
+    const unsigned base = stackOffset - CallFrame::headerSizeInRegisters;
+
+    const uint32_t gprLimit = base - stackCount - gprCount;
+    const uint32_t fprLimit = gprLimit - fprCount;
+
+    stackIndex = base;
+    gprIndex = base - stackCount;
+    fprIndex = gprIndex - gprCount;
+    for (uint32_t i = 0; i < signature.argumentCount(); i++) {
+        switch (signature.argument(i)) {
+        case Type::I32:
+        case Type::I64:
+        case Type::Anyref:
+        case Type::Funcref:
+            if (gprIndex > gprLimit)
+                arguments[i] = virtualRegisterForLocal(--gprIndex);
+            else
+                arguments[i] = virtualRegisterForLocal(--stackIndex);
+            break;
+        case Type::F32:
+        case Type::F64:
+            if (fprIndex > fprLimit)
+                arguments[i] = virtualRegisterForLocal(--fprIndex);
+            else
+                arguments[i] = virtualRegisterForLocal(--stackIndex);
+            break;
+        case Void:
+        case Func:
+            RELEASE_ASSERT_NOT_REACHED();
+        }
+    }
+
+    stackIndex = base;
+    gprIndex = base - stackCount;
+    fprIndex = gprIndex - gprCount;
+    for (uint32_t i = 0; i < signature.returnCount(); i++) {
+        switch (signature.returnType(i)) {
+        case Type::I32:
+        case Type::I64:
+        case Type::Anyref:
+        case Type::Funcref:
+            if (gprIndex > gprLimit)
+                temporaryResults[i] = virtualRegisterForLocal(--gprIndex);
+            else
+                temporaryResults[i] = virtualRegisterForLocal(--stackIndex);
+            break;
+        case Type::F32:
+        case Type::F64:
+            if (fprIndex > fprLimit)
+                temporaryResults[i] = virtualRegisterForLocal(--fprIndex);
+            else
+                temporaryResults[i] = virtualRegisterForLocal(--stackIndex);
+            break;
+        case Void:
+        case Func:
+            RELEASE_ASSERT_NOT_REACHED();
+        }
+    }
+
+    m_stackSize = initialStackSize;
+
+    auto commitResults = [this, temporaryResults = WTFMove(temporaryResults)](ExpressionList& results) {
+        for (auto temporaryResult : temporaryResults) {
+            ExpressionType result = push();
+            WasmMov::emit(this, result, temporaryResult);
+            results.append(result);
+        }
+    };
+
+    return LLIntCallInformation { stackOffset, stackCount, WTFMove(arguments), WTFMove(commitResults) };
+}
+
+auto LLIntGenerator::callInformationForCallee(const Signature& signature) -> Vector<VirtualRegister, 2>
+{
+    if (m_results.size())
+        return m_results;
+
+    m_results.reserveInitialCapacity(signature.returnCount());
+
+    const auto& callingConvention = wasmCallingConvention();
+    const uint32_t gprCount = callingConvention.gprArgs.size();
+    const uint32_t fprCount = callingConvention.fprArgs.size();
+
+    uint32_t gprIndex = 0;
+    uint32_t fprIndex = gprCount;
+    uint32_t stackIndex = 0;
+    const uint32_t maxGPRIndex = gprCount;
     const uint32_t maxFPRIndex = maxGPRIndex + fprCount;
-    uint32_t stackIndex = 0;
-    auto appendForType = [&](Type type, unsigned index, auto& vector) {
-        switch (type) {
+
+    for (uint32_t i = 0; i < signature.returnCount(); i++) {
+        switch (signature.returnType(i)) {
         case Type::I32:
         case Type::I64:
@@ -378 +618 @@
         case Type::Funcref:
             if (gprIndex < maxGPRIndex)
-                vector[index] = registers[registers.size() - gprIndex++ - 1];
-            else {
-                if (role == CallRole::Caller)
-                    vector[index] = registers[registers.size() - stackIndex++ - 1];
-                else
-                    vector[index] = new RegisterID(virtualRegisterForArgument(stackIndex++));
-            }
+                m_results.append(virtualRegisterForLocal(numberOfLLIntCalleeSaveRegisters + gprIndex++));
+            else
+                m_results.append(virtualRegisterForArgument(stackIndex++));
             break;
         case Type::F32:
         case Type::F64:
             if (fprIndex < maxFPRIndex)
-                vector[index] = registers[registers.size() - fprIndex++ - 1];
-            else {
-                if (role == CallRole::Caller)
-                    vector[index] = registers[registers.size() - stackIndex++ - 1];
-                else
-                    vector[index] = new RegisterID(virtualRegisterForArgument(stackIndex++));
-            }
+                m_results.append(virtualRegisterForLocal(numberOfLLIntCalleeSaveRegisters + fprIndex++));
+            else
+                m_results.append(virtualRegisterForArgument(stackIndex++));
             break;
         case Void:
@@ -401 +633 @@
             RELEASE_ASSERT_NOT_REACHED();
         }
-    };
-
-    gprIndex = stackCount;
-    fprIndex = maxGPRIndex;
-    for (uint32_t i = 0; i < signature.argumentCount(); i++)
-        appendForType(signature.argument(i), i, arguments);
-    gprIndex = stackCount;
-    fprIndex = maxGPRIndex;
-    for (uint32_t i = 0; i < signature.returnCount(); i++)
-        appendForType(signature.returnType(i), i, results);
-
-    if (role == CallRole::Caller) {
-        // Reserve space for call frame.
-        Vector<RefPtr<RegisterID>, CallFrame::headerSizeInRegisters + 2, UnsafeVectorOverflow> callFrame;
-        for (int i = 0; i < CallFrame::headerSizeInRegisters; ++i)
-            callFrame.append(newTemporary());
-    }
-
-    return LLIntCallInformation { stackOffset, stackCount, WTFMove(arguments), WTFMove(results) };
+    }
+
+    return m_results;
 }
 
@@ -436 +652 @@
     uint32_t stackIndex = 0;
 
-    Vector<RefPtr<RegisterID>> registerArguments(gprCount + fprCount);
+    Vector<VirtualRegister> registerArguments(gprCount + fprCount);
     for (uint32_t i = 0; i < gprCount + fprCount; i++)
-        registerArguments[i] = addVar();
+        registerArguments[i] = push();
 
     const auto addArgument = [&](uint32_t index, uint32_t& count, uint32_t max) {
@@ -465 +681 @@
     }
 
+    m_codeBlock->m_numVars += gprCount + fprCount;
+
     return { };
 }
@@ -470 +688 @@
 auto LLIntGenerator::addLocal(Type type, uint32_t count) -> PartialResult
 {
+    m_codeBlock->m_numVars += count;
     while (count--) {
-        auto local = addVar();
+        auto local = push();
         switch (type) {
         case Type::Anyref:
@@ -486 +705 @@
 void LLIntGenerator::didFinishParsingLocals()
 {
+    if (m_unitializedLocals.isEmpty())
+        return;
+
     auto null = jsNullConstant();
     for (auto local : m_unitializedLocals)
@@ -492 +714 @@
 }
 
-auto LLIntGenerator::addConstant(Type type, uint64_t value) -> ExpressionType
-{
-    VirtualRegister source(FirstConstantRegisterIndex + m_codeBlock->m_constants.size());
-    auto result = m_constantMap.emplace(value, source);
-    if (result.second) {
+auto LLIntGenerator::addConstant(Type type, int64_t value) -> ExpressionType
+{
+    auto constant = [&] {
+        if (!value)
+            return zeroConstant();
+
+        if (value == JSValue::encode(jsNull()))
+            return jsNullConstant();
+
+        VirtualRegister source(FirstConstantRegisterIndex + m_codeBlock->m_constants.size());
+        auto result = m_constantMap.add(value, source);
+        if (!result.isNewEntry)
+            return result.iterator->value;
         m_codeBlock->m_constants.append(value);
         if (UNLIKELY(Options::dumpGeneratedWasmBytecodes()))
             m_codeBlock->m_constantTypes.append(type);
-    } else
-        source = result.first->second;
-    auto target = newTemporary();
-    WasmMov::emit(this, target, source);
-    return target;
+        return source;
+    };
+    // leave a hole if we need to materialize the constant
+    push();
+    return constant();
 }
 
 auto LLIntGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
 {
-    // FIXME: Remove unnecessary moves
-    // https://bugs.webkit.org/show_bug.cgi?id=203657
-    result = newTemporary();
-    WasmMov::emit(this, result, virtualRegisterForLocal(index));
+    // leave a hole if we need to materialize the local
+    push();
+    result = virtualRegisterForWasmLocal(index);
     return { };
 }
@@ -518 +747 @@
 auto LLIntGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
 {
-    WasmMov::emit(this, virtualRegisterForLocal(index), value);
+    VirtualRegister target = virtualRegisterForWasmLocal(index);
+
+    // If this local is currently on the stack we need to materialize it, otherwise it'll see the new value instead of the old one
+    walkExpressionStack(m_parser->expressionStack(), [&](VirtualRegister& expression, VirtualRegister slot) {
+        if (expression != target)
+            return;
+        WasmMov::emit(this, slot, expression);
+        expression = slot;
+    });
+
+    WasmMov::emit(this, target, value);
+
     return { };
 }
@@ -524 +764 @@
 auto LLIntGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
 {
-    result = newTemporary();
+    result = push();
     WasmGetGlobal::emit(this, result, index);
     return { };
@@ -541 +781 @@
 auto LLIntGenerator::addLoop(BlockSignature signature, Stack& enclosingStack, ControlType& block, Stack& newStack, uint32_t loopIndex) -> PartialResult
 {
+    newStack = splitStack(signature, enclosingStack);
+    materializeConstantsAndLocals(newStack);
+
     Ref<Label> body = newEmittedLabel();
     Ref<Label> continuation = newLabel();
 
-    newStack = splitStack(signature, enclosingStack);
-    block = ControlType::loop(signature, newStack, WTFMove(body), WTFMove(continuation));
+    block = ControlType::loop(signature, m_stackSize, WTFMove(body), WTFMove(continuation));
 
     Vector<VirtualRegister> osrEntryData;
@@ -555 +797 @@
     const uint32_t fprCount = callingConvention.fprArgs.size();
     for (int32_t i = gprCount + fprCount + numberOfLLIntCalleeSaveRegisters; i < m_codeBlock->m_numVars; i++)
-        osrEntryData.append(::JSC::virtualRegisterForLocal(i));
+        osrEntryData.append(virtualRegisterForLocal(i));
     for (unsigned controlIndex = 0; controlIndex < m_parser->controlStack().size(); ++controlIndex) {
-        ExpressionList& expressionStack = m_parser->controlStack()[controlIndex].enclosedExpressionStack;
+        Stack& expressionStack = m_parser->controlStack()[controlIndex].enclosedExpressionStack;
         for (auto& expression : expressionStack)
-            osrEntryData.append(expression->virtualRegister());
+            osrEntryData.append(expression);
     }
 
@@ -571 +813 @@
 auto LLIntGenerator::addTopLevel(BlockSignature signature) -> ControlType
 {
-    return ControlType::topLevel(signature, tmpsForSignature(signature), newLabel());
+    return ControlType::topLevel(signature, m_stackSize, newLabel());
 }
 
@@ -577 +819 @@
 {
     newStack = splitStack(signature, enclosingStack);
-    newBlock = ControlType::block(signature, tmpsForSignature(signature), newLabel());
+    newBlock = ControlType::block(signature, m_stackSize, newLabel());
     return { };
 }
@@ -586 +828 @@
     Ref<Label> continuation = newLabel();
 
+    newStack = splitStack(signature, enclosingStack);
+
     WasmJfalse::emit(this, condition, alternate->bind(this));
 
-    newStack = splitStack(signature, enclosingStack);
-    result = ControlType::if_(signature, tmpsForSignature(signature), WTFMove(alternate), WTFMove(continuation));
-    return { };
-}
-
-auto LLIntGenerator::addElse(ControlType& data, const ExpressionList& currentStack) -> PartialResult
+    result = ControlType::if_(signature, m_stackSize, WTFMove(alternate), WTFMove(continuation));
+    return { };
+}
+
+auto LLIntGenerator::addElse(ControlType& data, Stack& expressionStack) -> PartialResult
 {
     ASSERT(WTF::holds_alternative<ControlIf>(data));
-    unifyValuesWithBlock(data.m_results, currentStack);
+    materializeConstantsAndLocals(expressionStack);
     WasmJmp::emit(this, data.m_continuation->bind(this));
     return addElseToUnreachable(data);
@@ -603 +846 @@
 auto LLIntGenerator::addElseToUnreachable(ControlType& data) -> PartialResult
 {
+    m_stackSize = data.stackSize() + data.m_signature->argumentCount();
+
     ControlIf& control = WTF::get<ControlIf>(data);
     emitLabel(control.m_alternate.get());
-    data = ControlType::block(data.m_signature, WTFMove(data.m_results), WTFMove(data.m_continuation));
-    return { };
-}
-
-auto LLIntGenerator::addReturn(const ControlType& data, const ExpressionList& returnValues) -> PartialResult
+    data = ControlType::block(data.m_signature, m_stackSize, WTFMove(data.m_continuation));
+    return { };
+}
+
+auto LLIntGenerator::addReturn(const ControlType& data, Stack& returnValues) -> PartialResult
 {
     if (!data.m_signature->returnCount()) {
@@ -616 +861 @@
     }
 
-    LLIntCallInformation info = callInformationFor(*data.m_signature, CallRole::Callee);
-    unifyValuesWithBlock(info.results, returnValues);
+    // no need to drop keep here, since we have to move anyway
+    unifyValuesWithBlock(callInformationForCallee(*data.m_signature), returnValues);
     WasmRet::emit(this);
 
@@ -623 +868 @@
 }
 
-auto LLIntGenerator::addBranch(ControlType& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
-{
-    unifyValuesWithBlock(data.m_results, returnValues);
-
+auto LLIntGenerator::addBranch(ControlType& data, ExpressionType condition, Stack& returnValues) -> PartialResult
+{
     RefPtr<Label> target = data.targetLabelForBranch();
-    if (condition)
-        WasmJtrue::emit(this, condition, target->bind(this));
-    else
-        WasmJmp::emit(this, target->bind(this));
-
-    return { };
-}
-
-auto LLIntGenerator::addSwitch(ExpressionType condition, const Vector<ControlType*>& targets, ControlType& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
-{
+    RefPtr<Label> skip = nullptr;
+
+    materializeConstantsAndLocals(returnValues);
+
+    if (condition.isValid()) {
+        skip = newLabel();
+        WasmJfalse::emit(this, condition, skip->bind(this));
+    }
+
+    dropKeep(returnValues, data, !skip);
+    WasmJmp::emit(this, target->bind(this));
+
+    if (skip)
+        emitLabel(*skip);
+
+    return { };
+}
+
+auto LLIntGenerator::addSwitch(ExpressionType condition, const Vector<ControlType*>& targets, ControlType& defaultTarget, Stack& expressionStack) -> PartialResult
+{
+    materializeConstantsAndLocals(expressionStack);
+
     unsigned tableIndex = m_codeBlock->numberOfJumpTables();
-    FunctionCodeBlock::JumpTable& jumpTable = m_codeBlock->addJumpTable(targets.size());
-
-    for (const auto& target : targets)
-        unifyValuesWithBlock(target->m_results, expressionStack);
-    unifyValuesWithBlock(defaultTarget.m_results, expressionStack);
-
-    WasmSwitch::emit(this, condition, tableIndex, defaultTarget.targetLabelForBranch()->bind(this));
+    FunctionCodeBlock::JumpTable& jumpTable = m_codeBlock->addJumpTable(targets.size() + 1);
+
+    WasmSwitch::emit(this, condition, tableIndex);
 
     unsigned index = 0;
     InstructionStream::Offset offset = m_lastInstruction.offset();
-    for (const auto& target : targets) {
-        RefPtr<Label> targetLabel = target->targetLabelForBranch();
+
+    auto addTarget = [&](ControlType& target) {
+        RefPtr<Label> targetLabel = target.targetLabelForBranch();
+
+        getDropKeepCount(target, jumpTable[index].startOffset, jumpTable[index].dropCount, jumpTable[index].keepCount);
+
         if (targetLabel->isForward()) {
             auto result = m_switches.add(targetLabel.get(), Vector<SwitchEntry>());
-            ASSERT(!jumpTable[index]);
-            result.iterator->value.append({ offset, &jumpTable[index++] });
+            ASSERT(!jumpTable[index].target);
+            result.iterator->value.append(SwitchEntry { offset, &jumpTable[index++].target });
         } else {
-            int jumpTarget = targetLabel->bind(this).target();
+            int jumpTarget = targetLabel->location() - offset;
             ASSERT(jumpTarget);
-            jumpTable[index++] = jumpTarget;
-        }
-    }
-
-
-    return { };
-}
-
-auto LLIntGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
+            jumpTable[index++].target = jumpTarget;
+        }
+    };
+
+    for (const auto& target : targets)
+        addTarget(*target);
+    addTarget(defaultTarget);
+
+    return { };
+}
+
+auto LLIntGenerator::endBlock(ControlEntry& entry, Stack& expressionStack) -> PartialResult
+{
+    // FIXME: We only need to materialize constants here if there exists a jump to this label
+    // https://bugs.webkit.org/show_bug.cgi?id=203657
+    materializeConstantsAndLocals(expressionStack);
+    return addEndToUnreachable(entry, expressionStack, false);
+}
+
+
+auto LLIntGenerator::addEndToUnreachable(ControlEntry& entry, const Stack& expressionStack, bool unreachable) -> PartialResult
 {
     ControlType& data = entry.controlData;
 
-    if (!WTF::holds_alternative<ControlLoop>(data))
-        unifyValuesWithBlock(data.m_results, expressionStack);
-
-    return addEndToUnreachable(entry, expressionStack);
-}
-
-
-auto LLIntGenerator::addEndToUnreachable(ControlEntry& entry, const Stack& expressionStack) -> PartialResult
-{
-    ControlType& data = entry.controlData;
-
-    emitLabel(*data.m_continuation);
-
-    if (!WTF::holds_alternative<ControlLoop>(data))
-        entry.enclosedExpressionStack.appendVector(data.m_results);
-    else {
-        for (unsigned i = 0; i < data.m_signature->returnCount(); ++i) {
-            if (i < expressionStack.size())
-                entry.enclosedExpressionStack.append(expressionStack[i]);
-            else
-                entry.enclosedExpressionStack.append(newTemporary());
-        }
-    }
-
-    // TopLevel does not have any code after this so we need to make sure we emit a return here.
-    if (WTF::holds_alternative<ControlTopLevel>(data))
-        return addReturn(data, entry.enclosedExpressionStack);
+    RELEASE_ASSERT(unreachable || m_stackSize == data.stackSize() + data.m_signature->returnCount());
+
+    m_stackSize = data.stackSize();
+
+    for (unsigned i = 0; i < data.m_signature->returnCount(); ++i) {
+        // We don't want to do a consistency check here because we just reset the stack size
+        // are pushing new values, while we already have the same values in the stack.
+        // The only reason we do things this way is so that it also works for unreachable blocks,
+        // since they might not have the right number of values in the expression stack.
+        // Instead, we do a stricter consistency check below.
+        auto tmp = push(NoConsistencyCheck);
+        ASSERT_UNUSED(expressionStack, unreachable || tmp == expressionStack[i]);
+        entry.enclosedExpressionStack.append(tmp);
+    }
+
+    if (m_lastOpcodeID == wasm_jmp && data.m_continuation->unresolvedJumps().size() == 1 && data.m_continuation->unresolvedJumps()[0] == static_cast<int>(m_lastInstruction.offset())) {
+        m_lastOpcodeID = wasm_unreachable;
+        m_writer.rewind(m_lastInstruction);
+    } else
+        emitLabel(*data.m_continuation);
+
+    return { };
+}
+
+auto LLIntGenerator::endTopLevel(BlockSignature signature, const Stack& expressionStack) -> PartialResult
+{
+    RELEASE_ASSERT(expressionStack.size() == signature->returnCount());
+
+    if (!signature->returnCount()) {
+        WasmRetVoid::emit(this);
+        return { };
+    }
+
+    checkConsistency();
+    unifyValuesWithBlock(callInformationForCallee(*signature), expressionStack);
+    WasmRet::emit(this);
 
     return { };
@@ -704 +980 @@
 {
     ASSERT(signature.argumentCount() == args.size());
-    LLIntCallInformation info = callInformationFor(signature);
+    LLIntCallInformation info = callInformationForCaller(signature);
     unifyValuesWithBlock(info.arguments, args);
-    results = WTFMove(info.results);
     if (Context::useFastTLS())
         WasmCall::emit(this, functionIndex, info.stackOffset, info.numberOfStackArguments);
     else
         WasmCallNoTls::emit(this, functionIndex, info.stackOffset, info.numberOfStackArguments);
+    info.commitResults(results);
 
     return { };
@@ -723 +999 @@
     ASSERT(m_info.tables[tableIndex].type() == TableElementType::Funcref);
 
-    LLIntCallInformation info = callInformationFor(signature);
+    LLIntCallInformation info = callInformationForCaller(signature);
     unifyValuesWithBlock(info.arguments, args);
-    results = WTFMove(info.results);
     if (Context::useFastTLS())
         WasmCallIndirect::emit(this, calleeIndex, m_codeBlock->addSignature(signature), info.stackOffset, info.numberOfStackArguments, tableIndex);
     else
         WasmCallIndirectNoTls::emit(this, calleeIndex, m_codeBlock->addSignature(signature), info.stackOffset, info.numberOfStackArguments, tableIndex);
+    info.commitResults(results);
 
     return { };
@@ -736 +1012 @@
 auto LLIntGenerator::addRefIsNull(ExpressionType value, ExpressionType& result) -> PartialResult
 {
-    result = value;
+    result = push();
     WasmRefIsNull::emit(this, result, value);
 
@@ -744 +1020 @@
 auto LLIntGenerator::addRefFunc(uint32_t index, ExpressionType& result) -> PartialResult
 {
-    result = newTemporary();
+    result = push();
     WasmRefFunc::emit(this, result, index);
 
@@ -752 +1028 @@
 auto LLIntGenerator::addTableGet(unsigned tableIndex, ExpressionType index, ExpressionType& result) -> PartialResult
 {
-    result = index;
+    result = push();
     WasmTableGet::emit(this, result, index, tableIndex);
 
@@ -767 +1043 @@
 auto LLIntGenerator::addTableSize(unsigned tableIndex, ExpressionType& result) -> PartialResult
 {
-    result = newTemporary();
+    result = push();
     WasmTableSize::emit(this, result, tableIndex);
 
@@ -775 +1051 @@
 auto LLIntGenerator::addTableGrow(unsigned tableIndex, ExpressionType fill, ExpressionType delta, ExpressionType& result) -> PartialResult
 {
-    result = fill;
+    result = push();
     WasmTableGrow::emit(this, result, fill, delta, tableIndex);
 
@@ -797 +1073 @@
 auto LLIntGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
 {
-    result = newTemporary();
+    result = push();
     WasmCurrentMemory::emit(this, result);
 
@@ -805 +1081 @@
 auto LLIntGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
 {
-    result = delta;
+    result = push();
     WasmGrowMemory::emit(this, result, delta);
 
@@ -813 +1089 @@
 auto LLIntGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
 {
-    result = condition;
+    result = push();
     WasmSelect::emit(this, result, condition, nonZero, zero);
 
@@ -821 +1097 @@
 auto LLIntGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
 {
-    result = pointer;
+    result = push();
     switch (op) {
     case LoadOpType::I32Load8S:
@@ -933 +1209 @@
         CASE(WasmJtrue)
         CASE(WasmJfalse)
-        case WasmSwitch::opcodeID: {
-            ASSERT((!instruction->as<WasmSwitch, WasmOpcodeTraits>().m_defaultTarget));
-            instruction->cast<WasmSwitch, WasmOpcodeTraits>()->setDefaultTarget(BoundLabel(target), [&]() {
-                generator.m_codeBlock->addOutOfLineJumpTarget(instruction.offset(), target);
-                return BoundLabel();
-            });
-            break;
-        }
         default:
-            ASSERT_NOT_REACHED();
+            RELEASE_ASSERT_NOT_REACHED();
         }
 #undef CASE

trunk/Source/JavaScriptCore/wasm/WasmLLIntPlan.cpp

@@ -81 +81 @@
 
     if (m_exportedFunctionIndices.contains(functionIndex) || m_moduleInformation->referencedFunctions().contains(functionIndex)) {
-        auto locker = holdLock(m_lock);
         EmbederToWasmFunction entry;
         entry.jit = makeUnique<CCallHelpers>();
         entry.function = m_createEmbedderWrapper(*entry.jit, signature, &m_unlinkedWasmToWasmCalls[functionIndex], m_moduleInformation.get(), m_mode, functionIndex);
+        auto locker = holdLock(m_lock);
         auto result = m_embedderToWasmInternalFunctions.add(functionIndex, WTFMove(entry));
         ASSERT_UNUSED(result, result.isNewEntry);

trunk/Source/JavaScriptCore/wasm/WasmValidate.cpp

@@ -162 +162 @@
 
     Result WARN_UNUSED_RETURN addUnreachable() { return { }; }
+    Result WARN_UNUSED_RETURN endTopLevel(BlockSignature, const Stack&) { return { }; }
 
     // Calls
@@ -177 +178 @@
     void setParser(FunctionParser<Validate>*) { }
     void didFinishParsingLocals() { }
+    void didPopValueFromStack() { }
 
 private: