diff --git a/CMakeLists.txt b/CMakeLists.txt index 49275bea2..e48194f4a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,9 @@ option(ZEN_ENABLE_DUMP_CALL_STACK "Enable exception call stack dump" OFF) option(ZEN_ENABLE_EVM_GAS_REGISTER "Enable gas register optimization for x86_64 multipass JIT" OFF ) +option(ZEN_ENABLE_EVM_STACK_SSA_LIFT + "Enable conservative EVM stack lifting in multipass frontend" OFF +) # Blockchain options option(ZEN_ENABLE_CHECKED_ARITHMETIC "Enable checked arithmetic" OFF) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index bbe49d442..5d73d028c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -73,6 +73,10 @@ if(ZEN_ENABLE_JIT_PRECOMPILE_FALLBACK) add_definitions(-DZEN_ENABLE_JIT_PRECOMPILE_FALLBACK) endif() +if(ZEN_ENABLE_EVM_STACK_SSA_LIFT) + add_definitions(-DZEN_ENABLE_EVM_STACK_SSA_LIFT) +endif() + if(ZEN_ENABLE_CPU_EXCEPTION) if(ZEN_ENABLE_SINGLEPASS_JIT OR ZEN_ENABLE_MULTIPASS_JIT) add_definitions(-DZEN_ENABLE_CPU_EXCEPTION) diff --git a/src/action/evm_bytecode_visitor.h b/src/action/evm_bytecode_visitor.h index fc639044d..95f1e2229 100644 --- a/src/action/evm_bytecode_visitor.h +++ b/src/action/evm_bytecode_visitor.h @@ -5,23 +5,33 @@ #define ZEN_ACTION_EVM_BYTECODE_VISITOR_H #include "compiler/evm_frontend/evm_analyzer.h" +#include "compiler/evm_frontend/evm_lifted_stack_lifter.h" #include "compiler/evm_frontend/evm_mir_compiler.h" #include "evmc/evmc.h" #include "evmc/instructions.h" #include "runtime/evm_module.h" +#include +#include +#include +#include +#include + namespace COMPILER { template class EVMByteCodeVisitor { typedef typename IRBuilder::CompilerContext CompilerContext; typedef typename IRBuilder::Operand Operand; typedef zen::action::VMEvalStack EvalStack; + using StackLifterType = EVMLiftedStackLifter; + using MergeMaterializationRequest = + typename StackLifterType::MergeMaterializationRequest; using Byte = zen::common::Byte; using Bytes = zen::common::Bytes; public: EVMByteCodeVisitor(IRBuilder &Builder, CompilerContext *Ctx) - : Builder(Builder), Ctx(Ctx) { + : Builder(Builder), Ctx(Ctx), StackLifter(Builder) { ZEN_ASSERT(Ctx); } @@ -34,16 +44,77 @@ template class EVMByteCodeVisitor { private: static constexpr size_t EVM_MAX_STACK_SIZE = 1024; + static constexpr size_t EVM_MAX_PUSH_IMMEDIATE_SIZE = 32; + + template + struct HasRegisterCurrentBlockPC : std::false_type {}; + template + struct HasRegisterCurrentBlockPC< + T, std::void_t().registerCurrentBlockPC( + uint64_t{}))>> : std::true_type {}; + + template + struct HasSpillTrackedStackPreservingPrefix : std::false_type {}; + template + struct HasSpillTrackedStackPreservingPrefix< + T, std::void_t< + decltype(std::declval().spillTrackedStackPreservingPrefix( + std::declval &>(), uint32_t{}))>> + : std::true_type {}; + + template + struct HasMaterializeStackMergeOperand : std::false_type {}; + template + struct HasMaterializeStackMergeOperand< + T, + std::void_t().materializeStackMergeOperand( + std::declval &>(), + std::declval> &>()))>> + : std::true_type {}; + + void registerCurrentBlockPC(uint64_t BlockPC) { + if constexpr (HasRegisterCurrentBlockPC::value) { + Builder.registerCurrentBlockPC(BlockPC); + } else { + (void)BlockPC; + } + } - void push(const Operand &Opnd) { Stack.push(Opnd); } + void spillTrackedStackPreservingPrefix(const std::vector &Values, + uint32_t PrefixDepth) { + if constexpr (HasSpillTrackedStackPreservingPrefix::value) { + Builder.spillTrackedStackPreservingPrefix(Values, PrefixDepth); + } else { + (void)PrefixDepth; + Builder.spillTrackedStack(Values); + } + } - Operand pop() { - Operand Opnd; - if (Stack.empty()) { - Opnd = Builder.stackPop(); + Operand materializeStackMergeOperandCompat( + const std::vector &PredBlockPCs, + const std::vector> &IncomingValues) { + if constexpr (HasMaterializeStackMergeOperand::value) { + return Builder.materializeStackMergeOperand(PredBlockPCs, IncomingValues); } else { - Opnd = Stack.pop(); + (void)PredBlockPCs; + Operand Result = Builder.createStackEntryOperand(); + if (!IncomingValues.empty()) { + Builder.assignStackEntryOperand(Result, IncomingValues.back().second); + } + return Result; } + } + + void push(const Operand &Opnd) { Stack.push(Opnd); } + + void requireLogicalStackDepth(uint32_t Depth) { + ZEN_ASSERT(Stack.getSize() >= Depth && + "Logical EVM stack must be preloaded at block entry"); + } + + Operand pop() { + requireLogicalStackDepth(1); + Operand Opnd = Stack.pop(); Builder.releaseOperand(Opnd); return Opnd; } @@ -55,6 +126,7 @@ template class EVMByteCodeVisitor { size_t BytecodeSize = Ctx->getBytecodeSize(); EVMAnalyzer Analyzer(Ctx->getRevision()); Analyzer.analyze(Bytecode, BytecodeSize); + initializeLiftedBlocks(Analyzer); const uint8_t *Ip = Bytecode; const bool StartsWithJumpDest = @@ -573,7 +645,45 @@ template class EVMByteCodeVisitor { // Control flow operations case OP_JUMP: { Operand Dest = pop(); - handleEndBlock(); + uint64_t SuccPC = 0; + bool HasLiftedSucc = tryAssignConstantJumpEntryState(Analyzer, Dest); + if (!HasLiftedSucc) { + if (CurrentBlockLifted) { + const bool HasKnownSucc = + tryGetConstantJumpSuccessorPC(Analyzer, Dest, SuccPC); + const bool HasKnownLiftedSucc = + HasKnownSucc && isLiftedBlock(SuccPC); + auto OutgoingStack = drainLogicalStack(); + if (HasKnownLiftedSucc) { + assignLiftedEntryState(SuccPC, OutgoingStack); + } + if (!HasKnownSucc) { + assignCompatibleDynamicJumpRegionEntryStates(Analyzer, + OutgoingStack); + } + const bool HasCompatibleDynamicTargets = + !HasKnownSucc && + !Analyzer + .getCompatibleDynamicJumpTargetBlocksForSourceBlock( + CurrentBlockEntryPC) + .empty(); + const bool NeedsRuntimeMaterialization = + (HasKnownSucc && !HasKnownLiftedSucc) || + (!HasKnownSucc && !HasCompatibleDynamicTargets); + finalizeBlockExit(std::move(OutgoingStack), + NeedsRuntimeMaterialization); + } else { + handleEndBlock(); + if (!tryGetConstantJumpSuccessorPC(Analyzer, Dest, SuccPC)) { + assignCompatibleDynamicJumpRegionEntryStatesFromRuntime( + Analyzer); + } + } + if (tryGetConstantJumpSuccessorPC(Analyzer, Dest, SuccPC) && + isLiftedBlock(SuccPC)) { + assignLiftedEntryStateFromRuntime(Analyzer, SuccPC); + } + } Builder.handleJump(Dest); break; } @@ -581,8 +691,67 @@ template class EVMByteCodeVisitor { case OP_JUMPI: { Operand Dest = pop(); Operand Cond = pop(); - handleEndBlock(); + uint64_t JumpSuccPC = 0; + bool HasJumpSucc = + tryGetConstantJumpSuccessorPC(Analyzer, Dest, JumpSuccPC); + uint64_t FallthroughPC = PC + 1; + if (Analyzer.hasCanonicalJumpDest(FallthroughPC)) { + FallthroughPC = Analyzer.getCanonicalJumpDestPC(FallthroughPC); + } + bool CanLiftFallthrough = + CurrentBlockLifted && isLiftedBlock(FallthroughPC); + bool CanLiftJump = + HasJumpSucc && CurrentBlockLifted && isLiftedBlock(JumpSuccPC); + bool CanPreassignFallthrough = + CurrentBlockLifted && isLiftedBlock(FallthroughPC); + bool CanPreassignJump = + CurrentBlockLifted && HasJumpSucc && isLiftedBlock(JumpSuccPC); + bool CanTransferWithoutMaterialize = + CurrentBlockLifted && CanLiftFallthrough && CanLiftJump; + + if (CanTransferWithoutMaterialize) { + auto OutgoingStack = drainLogicalStack(); + assignLiftedEntryState(FallthroughPC, OutgoingStack); + assignLiftedEntryState(JumpSuccPC, OutgoingStack); + finalizeBlockExit(std::move(OutgoingStack), false); + } else { + if (CurrentBlockLifted) { + auto OutgoingStack = drainLogicalStack(); + if (CanPreassignFallthrough) { + assignLiftedEntryState(FallthroughPC, OutgoingStack); + } + if (CanPreassignJump) { + assignLiftedEntryState(JumpSuccPC, OutgoingStack); + } + if (!HasJumpSucc) { + assignCompatibleDynamicJumpRegionEntryStates(Analyzer, + OutgoingStack); + } + bool NeedsRuntimeMaterialization = !CanPreassignFallthrough; + if (!NeedsRuntimeMaterialization) { + if (HasJumpSucc) { + NeedsRuntimeMaterialization = !CanPreassignJump; + } + } + finalizeBlockExit(std::move(OutgoingStack), + NeedsRuntimeMaterialization); + } else { + handleEndBlock(); + if (isLiftedBlock(FallthroughPC)) { + assignLiftedEntryStateFromRuntime(Analyzer, FallthroughPC); + } + if (HasJumpSucc) { + if (isLiftedBlock(JumpSuccPC)) { + assignLiftedEntryStateFromRuntime(Analyzer, JumpSuccPC); + } + } else { + assignCompatibleDynamicJumpRegionEntryStatesFromRuntime( + Analyzer); + } + } + } Builder.handleJumpI(Dest, Cond); + PC = FallthroughPC; handleBeginBlock(Analyzer); break; } @@ -591,18 +760,33 @@ template class EVMByteCodeVisitor { // Consecutive JUMPDEST opcodes share one body BB in multipass. // Charge all skipped metering points before jumping to the shared // destination at the end of the run. + bool HasLiveFallthrough = !InDeadCode; uint64_t RunStartPC = PC; while (Ip < IpEnd && static_cast(*Ip) == OP_JUMPDEST) { Ip++; PC++; } - if (PC > RunStartPC && !InDeadCode) { + if (PC > RunStartPC && HasLiveFallthrough) { Builder.meterOpcodeRange(RunStartPC, PC); } - handleEndBlock(); + if (HasLiveFallthrough && + tryAssignFallthroughEntryState(Analyzer, PC)) { + // Keep runtime stack materialization elided on lifted fallthrough. + } else { + if (HasLiveFallthrough && CurrentBlockLifted && isLiftedBlock(PC)) { + auto OutgoingStack = drainLogicalStack(); + assignLiftedEntryState(PC, OutgoingStack); + finalizeBlockExit(std::move(OutgoingStack), false); + } else { + handleEndBlock(); + if (HasLiveFallthrough && isLiftedBlock(PC)) { + assignLiftedEntryStateFromRuntime(Analyzer, PC); + } + } + } Builder.handleJumpDest(PC); - Builder.meterOpcode(Opcode, PC); handleBeginBlock(Analyzer); + Builder.meterOpcode(Opcode, PC); break; } @@ -650,6 +834,7 @@ template class EVMByteCodeVisitor { PC++; // offset 1 byte for opcode } if (!InDeadCode) { + handleEndBlock(); handleStop(); } } catch (const common::Error &E) { @@ -659,28 +844,244 @@ template class EVMByteCodeVisitor { return true; } + void initializeLiftedBlocks(const EVMAnalyzer &Analyzer) { + StackLifter.initialize(Analyzer); + } + + bool isLiftedBlock(uint64_t BlockPC) const { + return StackLifter.isLiftedBlock(BlockPC); + } + + bool canAssignLiftedEntryStateFromRuntime(const EVMAnalyzer &Analyzer, + uint64_t PredBlockPC, + uint64_t SuccBlockPC) const { + if (!isLiftedBlock(SuccBlockPC)) { + return false; + } + + const auto &BlockInfos = Analyzer.getBlockInfos(); + auto PredIt = BlockInfos.find(PredBlockPC); + auto SuccIt = BlockInfos.find(SuccBlockPC); + if (PredIt == BlockInfos.end() || SuccIt == BlockInfos.end()) { + return false; + } + + return PredIt->second.ResolvedExitStackDepth >= 0 && + SuccIt->second.FullEntryStateDepth >= 0 && + PredIt->second.ResolvedExitStackDepth == + SuccIt->second.FullEntryStateDepth; + } + + std::vector drainLogicalStack() { + EvalStack ReverseStack; + std::vector Values; + while (!Stack.empty()) { + ReverseStack.push(Stack.pop()); + } + while (!ReverseStack.empty()) { + Values.push_back(ReverseStack.pop()); + } + return Values; + } + + void restoreLogicalStack(const std::vector &Values) { + for (const Operand &Opnd : Values) { + Stack.push(Opnd); + } + } + + void finalizeBlockExit(std::vector Values, bool Materialize) { + if (Materialize) { + if (CurrentBlockLifted) { + spillTrackedStackPreservingPrefix(Values, + CurrentBlockHiddenLiveInPrefixDepth); + } else { + for (const Operand &Opnd : Values) { + Builder.stackPush(Opnd); + } + } + } + InDeadCode = true; + CurrentBlockLifted = false; + CurrentBlockHiddenLiveInPrefixDepth = 0; + } + + bool tryGetConstantJumpSuccessorPC(const EVMAnalyzer &Analyzer, + const Operand &Dest, + uint64_t &SuccPC) const { + if (!Dest.isConstant()) { + return false; + } + const auto &ConstValue = Dest.getConstValue(); + if ((ConstValue[3] | ConstValue[2] | ConstValue[1]) != 0) { + return false; + } + uint64_t RawDest = ConstValue[0]; + if (!Analyzer.hasCanonicalJumpDest(RawDest)) { + return false; + } + SuccPC = Analyzer.getCanonicalJumpDestPC(RawDest); + return true; + } + + void assignLiftedEntryState(uint64_t BlockPC, + const std::vector &Values) { + StackLifter.assignEntryState(CurrentBlockEntryPC, BlockPC, Values); + } + + void assignCompatibleDynamicJumpRegionEntryStates( + const EVMAnalyzer &Analyzer, const std::vector &Values) { + for (uint64_t TargetBlockPC : + Analyzer.getCompatibleDynamicJumpTargetBlocksForSourceBlock( + CurrentBlockEntryPC)) { + if (!isLiftedBlock(TargetBlockPC)) { + continue; + } + StackLifter.assignEntryState(CurrentBlockEntryPC, TargetBlockPC, Values); + } + } + + void assignCompatibleDynamicJumpRegionEntryStatesFromRuntime( + const EVMAnalyzer &Analyzer) { + for (uint64_t TargetBlockPC : + Analyzer.getCompatibleDynamicJumpTargetBlocksForSourceBlock( + CurrentBlockEntryPC)) { + if (!canAssignLiftedEntryStateFromRuntime(Analyzer, CurrentBlockEntryPC, + TargetBlockPC)) { + continue; + } + StackLifter.assignEntryState( + CurrentBlockEntryPC, TargetBlockPC, + loadLiftedEntryStateFromRuntime(Analyzer, TargetBlockPC)); + } + } + + void assignLiftedEntryStateFromRuntime(const EVMAnalyzer &Analyzer, + uint64_t BlockPC) { + if (!canAssignLiftedEntryStateFromRuntime(Analyzer, CurrentBlockEntryPC, + BlockPC)) { + return; + } + StackLifter.assignEntryState( + CurrentBlockEntryPC, BlockPC, + loadLiftedEntryStateFromRuntime(Analyzer, BlockPC)); + } + + bool tryAssignConstantJumpEntryState(const EVMAnalyzer &Analyzer, + const Operand &Dest) { + uint64_t SuccPC = 0; + if (!CurrentBlockLifted || + !tryGetConstantJumpSuccessorPC(Analyzer, Dest, SuccPC) || + !isLiftedBlock(SuccPC)) { + return false; + } + auto OutgoingStack = drainLogicalStack(); + assignLiftedEntryState(SuccPC, OutgoingStack); + finalizeBlockExit(std::move(OutgoingStack), false); + return true; + } + + bool tryAssignFallthroughEntryState(const EVMAnalyzer &Analyzer, + uint64_t SuccPC) { + (void)Analyzer; + if (!CurrentBlockLifted || !isLiftedBlock(SuccPC)) { + return false; + } + auto OutgoingStack = drainLogicalStack(); + assignLiftedEntryState(SuccPC, OutgoingStack); + finalizeBlockExit(std::move(OutgoingStack), false); + return true; + } + + std::vector + loadLiftedEntryStateFromRuntime(const EVMAnalyzer &Analyzer, + uint64_t BlockPC) { + std::vector Values; + const auto &BlockInfos = Analyzer.getBlockInfos(); + auto It = BlockInfos.find(BlockPC); + if (It == BlockInfos.end() || !isLiftedBlock(BlockPC)) { + return Values; + } + const auto &BlockInfo = It->second; + ZEN_ASSERT(BlockInfo.ResolvedEntryStackDepth >= 0 && + "Lifted block must have resolved entry depth"); + ZEN_ASSERT(BlockInfo.FullEntryStateDepth >= 0 && + "Lifted block must have full entry state depth"); + Values.reserve(static_cast(BlockInfo.FullEntryStateDepth)); + for (int32_t Index = 0; Index < BlockInfo.FullEntryStateDepth; ++Index) { + int32_t StackIndex = BlockInfo.ResolvedEntryStackDepth - Index - 1; + Values.push_back(Builder.stackGet(StackIndex)); + } + return Values; + } + + bool validateLiftedBlockStackBounds(const EVMAnalyzer::BlockInfo &BlockInfo) { + ZEN_ASSERT(BlockInfo.ResolvedEntryStackDepth >= 0 && + "Lifted block must have resolved entry depth"); + + int64_t EntryDepth = + static_cast(BlockInfo.ResolvedEntryStackDepth); + int64_t MinDepth = + EntryDepth + static_cast(BlockInfo.MinStackHeight); + if (MinDepth < 0) { + Builder.handleTrap(common::ErrorCode::EVMStackUnderflow); + InDeadCode = true; + CurrentBlockLifted = false; + return false; + } + + int64_t MaxDepth = + EntryDepth + static_cast(BlockInfo.MaxStackHeight); + if (MaxDepth > static_cast(EVM_MAX_STACK_SIZE)) { + Builder.handleTrap(common::ErrorCode::EVMStackOverflow); + InDeadCode = true; + CurrentBlockLifted = false; + return false; + } + + return true; + } + void handleBeginBlock(EVMAnalyzer &Analyzer) { const auto &BlockInfos = Analyzer.getBlockInfos(); ZEN_ASSERT(BlockInfos.count(PC) > 0 && "Block info not found"); const auto &BlockInfo = BlockInfos.at(PC); - if (BlockInfo.HasUndefinedInstr) { - Builder.handleUndefined(); - InDeadCode = true; + CurrentBlockEntryPC = PC; + CurrentBlockHiddenLiveInPrefixDepth = 0; + registerCurrentBlockPC(PC); + bool LiftedBlock = isLiftedBlock(PC); + if (LiftedBlock && !validateLiftedBlockStackBounds(BlockInfo)) { return; } + if (static_cast(-BlockInfo.MinStackHeight) > EVM_MAX_STACK_SIZE) { Builder.handleTrap(common::ErrorCode::EVMStackUnderflow); InDeadCode = true; + CurrentBlockLifted = false; return; } if (static_cast(BlockInfo.MaxStackHeight) > EVM_MAX_STACK_SIZE) { Builder.handleTrap(common::ErrorCode::EVMStackOverflow); InDeadCode = true; + CurrentBlockLifted = false; return; } InDeadCode = false; - Builder.createStackCheckBlock(-BlockInfo.MinStackHeight, - 1024 - BlockInfo.MaxStackHeight); + if (!LiftedBlock) { + Builder.createStackCheckBlock(-BlockInfo.MinStackHeight, + 1024 - BlockInfo.MaxStackHeight); + } + + if (LiftedBlock) { + CurrentBlockLifted = true; + CurrentBlockHiddenLiveInPrefixDepth = + static_cast(std::max(BlockInfo.HiddenLiveInPrefixDepth, 0)); + materializeLiftedBlockMergeRequests(PC); + restoreLiftedBlockLogicalEntryState(PC); + return; + } + + CurrentBlockLifted = false; int32_t TotalPopSize = -BlockInfo.MinPopHeight; EvalStack ReverseStack; while (TotalPopSize > 0) { @@ -693,20 +1094,32 @@ template class EVMByteCodeVisitor { } } - void handleEndBlock() { - // Save unused stack elements to runtime - EvalStack ReverseStack; - while (!Stack.empty()) { - Operand Opnd = Stack.pop(); - ReverseStack.push(Opnd); + void materializeLiftedBlockMergeRequests(uint64_t BlockPC) { + for (const MergeMaterializationRequest &Request : + StackLifter.getMergeMaterializationRequests(BlockPC)) { + std::vector> IncomingValues; + IncomingValues.reserve(Request.IncomingValues.size()); + for (const auto &IncomingValue : Request.IncomingValues) { + IncomingValues.emplace_back(IncomingValue.PredBlockPC, + IncomingValue.Value); + } + StackLifter.assignMergeOperand( + BlockPC, Request.SlotIndex, + materializeStackMergeOperandCompat(Request.ExpectedPredBlockPCs, + IncomingValues)); } - while (!ReverseStack.empty()) { - Operand Opnd = ReverseStack.pop(); - Builder.stackPush(Opnd); + } + + void restoreLiftedBlockLogicalEntryState(uint64_t BlockPC) { + std::vector LogicalEntryState = + StackLifter.getLogicalEntryState(BlockPC); + if (!LogicalEntryState.empty()) { + restoreLogicalStack(LogicalEntryState); } - InDeadCode = true; } + void handleEndBlock() { finalizeBlockExit(drainLogicalStack(), true); } + void handleStop() { Builder.handleStop(); } template void handleBinaryArithmetic() { @@ -835,22 +1248,26 @@ template class EVMByteCodeVisitor { uint64_t Available = (Start < BytecodeSize) ? (BytecodeSize - Start) : 0; uint64_t ReadCount = (Count < Available) ? Count : Available; - Bytes Result = ReadCount > 0 - ? Bytes(Bytecode + Start, static_cast(ReadCount)) - : Bytes(); + if (Count == 0) { + return Bytes(); + } + + ZEN_ASSERT(Count <= EVM_MAX_PUSH_IMMEDIATE_SIZE); + PushImmediateScratch.fill(Byte{0}); + for (uint64_t I = 0; I < ReadCount; ++I) { + PushImmediateScratch[static_cast(I)] = Bytecode[Start + I]; + } + PC += Count; - return Result; + return Bytes(PushImmediateScratch.data(), Count); } + std::array PushImmediateScratch = {}; + // DUP1-DUP16: Duplicate Nth stack item void handleDup(uint8_t Index) { - Operand Result; - if (Stack.getSize() < static_cast(Index)) { - int32_t MemIndex = static_cast(Index) - Stack.getSize() - 1; - Result = Builder.stackGet(MemIndex); - } else { - Result = Stack.peek(Index - 1); - } + requireLogicalStackDepth(Index); + Operand Result = Stack.peek(Index - 1); push(Result); } @@ -862,20 +1279,8 @@ template class EVMByteCodeVisitor { // SWAP1-SWAP16: Swap top with Nth+1 stack item void handleSwap(uint8_t Index) { - int32_t MemIndex = static_cast(Index) - Stack.getSize(); - if (Stack.empty()) { - Operand A = Builder.stackGet(0); - Operand B = Builder.stackGet(MemIndex); - Builder.stackSet(0, B); - Builder.stackSet(MemIndex, A); - } else if (Stack.getSize() < static_cast(Index) + 1u) { - Operand &A = Stack.peek(0); - Operand B = Builder.stackGet(MemIndex); - Builder.stackSet(MemIndex, A); - A = B; - } else { - std::swap(Stack.peek(0), Stack.peek(Index)); - } + requireLogicalStackDepth(static_cast(Index) + 1u); + std::swap(Stack.peek(0), Stack.peek(Index)); } // ==================== Environment Instruction Handlers ==================== @@ -980,8 +1385,12 @@ template class EVMByteCodeVisitor { IRBuilder &Builder; CompilerContext *Ctx; EvalStack Stack; + StackLifterType StackLifter; bool InDeadCode = false; uint64_t PC = 0; + uint64_t CurrentBlockEntryPC = 0; + bool CurrentBlockLifted = false; + uint32_t CurrentBlockHiddenLiveInPrefixDepth = 0; }; } // namespace COMPILER diff --git a/src/compiler/CMakeLists.txt b/src/compiler/CMakeLists.txt index fad9b680c..74f604ae4 100644 --- a/src/compiler/CMakeLists.txt +++ b/src/compiler/CMakeLists.txt @@ -61,6 +61,7 @@ set(COMPILER_SRCS cgir/pass/cg_register_class_info.cpp cgir/pass/prolog_epilog_inserter.cpp cgir/pass/expand_post_ra_pseudos.cpp + cgir/pass/phi_elimination.cpp cgir/pass/virt_reg_map.cpp cgir/pass/cg_dominators.cpp cgir/pass/cg_loop_info.cpp diff --git a/src/compiler/cgir/cg_basic_block.cpp b/src/compiler/cgir/cg_basic_block.cpp index 463ad4848..54bc3daf3 100644 --- a/src/compiler/cgir/cg_basic_block.cpp +++ b/src/compiler/cgir/cg_basic_block.cpp @@ -31,6 +31,57 @@ CgBasicBlock::const_parent_iterator CgBasicBlock::getIterator() const { return _parent->begin() + _idx; } +void CgBasicBlock::removeSuccessor(CgBasicBlock *Succ) { + auto It = std::find(Successors.begin(), Successors.end(), Succ); + removeSuccessor(It); +} + +void CgBasicBlock::removeSuccessor(succ_iterator It) { + ZEN_ASSERT(It != Successors.end()); + (*It)->removePredecessor(this); + Successors.erase(It); +} + +void CgBasicBlock::removePredecessor(CgBasicBlock *Pred) { + auto It = std::find(Predecessors.begin(), Predecessors.end(), Pred); + ZEN_ASSERT(It != Predecessors.end()); + Predecessors.erase(It); +} + +void CgBasicBlock::replaceSuccessor(CgBasicBlock *Old, CgBasicBlock *New) { + if (Old == New) { + return; + } + + auto E = Successors.end(); + auto OldIt = E; + auto NewIt = E; + for (auto It = Successors.begin(); It != E; ++It) { + if (*It == Old) { + OldIt = It; + if (NewIt != E) { + break; + } + } + if (*It == New) { + NewIt = It; + if (OldIt != E) { + break; + } + } + } + ZEN_ASSERT(OldIt != E && "Old is not a successor of this block"); + + if (NewIt == E) { + Old->removePredecessor(this); + New->addPredecessor(this); + *OldIt = New; + return; + } + + removeSuccessor(OldIt); +} + void CgBasicBlock::sortUniqueLiveIns() { llvm::sort(LiveIns, [](const RegisterMaskPair &LI0, const RegisterMaskPair &LI1) { diff --git a/src/compiler/cgir/cg_basic_block.h b/src/compiler/cgir/cg_basic_block.h index e56acb15e..61de674b6 100644 --- a/src/compiler/cgir/cg_basic_block.h +++ b/src/compiler/cgir/cg_basic_block.h @@ -218,11 +218,16 @@ class CgBasicBlock : public NonCopyable { Succ->addPredecessor(this); } + void removeSuccessor(CgBasicBlock *Succ); + void removeSuccessor(succ_iterator It); + bool isSuccessor(const CgBasicBlock *MBB) const { return is_contained(successors(), MBB); } void addPredecessor(CgBasicBlock *Pred) { Predecessors.push_back(Pred); } + void removePredecessor(CgBasicBlock *Pred); + void replaceSuccessor(CgBasicBlock *Old, CgBasicBlock *New); // Only call this method when you are certain that all blocks have been added // to the basic block list. diff --git a/src/compiler/cgir/cg_function.cpp b/src/compiler/cgir/cg_function.cpp index 7aee8b05a..f7f1460cd 100644 --- a/src/compiler/cgir/cg_function.cpp +++ b/src/compiler/cgir/cg_function.cpp @@ -79,6 +79,19 @@ CgInstruction *CgFunction::CloneMachineInstr(const CgInstruction *Orig) { CgInstruction(*this, *Orig); } +void CgFunction::insertCgBasicBlockAfter(CgBasicBlock *After, + CgBasicBlock *CgBB) { + ZEN_ASSERT(After != nullptr); + ZEN_ASSERT(CgBB != nullptr); + + const size_t InsertIndex = static_cast(After->getNumber()) + 1; + auto It = _cg_basic_blocks.begin() + InsertIndex; + _cg_basic_blocks.insert(It, CgBB); + for (size_t Index = InsertIndex; Index < _cg_basic_blocks.size(); ++Index) { + _cg_basic_blocks[Index]->setNumber(Index); + } +} + uint32_t CgFunction::createJumpTableIndex(const CompileVector &DestBBs) { ZEN_ASSERT(!DestBBs.empty() && "Cannot create an empty jump table!"); diff --git a/src/compiler/cgir/cg_function.h b/src/compiler/cgir/cg_function.h index 4029e0616..334b0e880 100644 --- a/src/compiler/cgir/cg_function.h +++ b/src/compiler/cgir/cg_function.h @@ -85,6 +85,8 @@ class CgFunction : public ContextObject { _cg_basic_blocks.emplace_back(CgBB); } + void insertCgBasicBlockAfter(CgBasicBlock *After, CgBasicBlock *CgBB); + CgBasicBlock *getCgBasicBlock(BlockNum BBIdx) const { ZEN_ASSERT(BBIdx < _cg_basic_blocks.size()); return _cg_basic_blocks[BBIdx]; diff --git a/src/compiler/cgir/lowering.h b/src/compiler/cgir/lowering.h index 2c90a5447..e83b599b1 100644 --- a/src/compiler/cgir/lowering.h +++ b/src/compiler/cgir/lowering.h @@ -103,6 +103,9 @@ template class CgLowering { void lowerStmt(const MInstruction &Inst) { switch (Inst.getKind()) { + case MInstruction::PHI: + lowerPhiStmt(llvm::cast(Inst)); + break; case MInstruction::DASSIGN: lowerDassignStmt(llvm::cast(Inst)); break; @@ -214,6 +217,11 @@ template class CgLowering { case MInstruction::SELECT: ResultReg = SELF.lowerSelectExpr(llvm::cast(Inst)); break; + case MInstruction::PHI: + ZEN_ASSERT(_expr_reg_map.count(&Inst) && + "phi must be lowered before it is used"); + ResultReg = _expr_reg_map[&Inst]; + break; case MInstruction::DREAD: ResultReg = lowerDreadExpr(llvm::cast(Inst)); break; @@ -240,6 +248,29 @@ template class CgLowering { var_reg); } + void lowerPhiStmt(const PhiInstruction &Inst) { + llvm::MVT VT = getMVT(*Inst.getType()); + CgRegister ResultReg = createReg(TLI.getRegClassFor(VT)); + std::vector Operands; + Operands.reserve(1 + Inst.getNumIncoming() * 2); + Operands.push_back(CgOperand::createRegOperand(ResultReg, true)); + + for (size_t Index = 0; Index < Inst.getNumIncoming(); ++Index) { + const MInstruction *IncomingValue = Inst.getIncomingValue(Index); + const MBasicBlock *IncomingBB = Inst.getIncomingBlock(Index); + ZEN_ASSERT(IncomingValue != nullptr); + ZEN_ASSERT(IncomingBB != nullptr); + CgRegister IncomingReg = lowerExpr(*IncomingValue); + Operands.push_back(CgOperand::createRegOperand(IncomingReg, false)); + Operands.push_back(CgOperand::createMBB(getOrCreateCgBB(IncomingBB))); + } + + llvm::MutableArrayRef OperandRef(Operands); + MF->createCgInstruction(*CurBB, TII.get(llvm::TargetOpcode::PHI), + OperandRef); + _expr_reg_map[&Inst] = ResultReg; + } + CgRegister lowerDreadExpr(const DreadInstruction &inst) { CgRegister ret; auto var_idx = inst.getVarIdx(); diff --git a/src/compiler/cgir/pass/phi_elimination.cpp b/src/compiler/cgir/pass/phi_elimination.cpp new file mode 100644 index 000000000..2990af917 --- /dev/null +++ b/src/compiler/cgir/pass/phi_elimination.cpp @@ -0,0 +1,206 @@ +// Copyright (C) 2025 the DTVM authors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "compiler/cgir/pass/phi_elimination.h" +#include "compiler/cgir/cg_basic_block.h" +#include "compiler/cgir/cg_function.h" +#include "compiler/cgir/pass/cg_register_info.h" +#include "compiler/llvm-prebuild/Target/X86/X86InstrInfo.h" +#include +#include +#include + +using namespace COMPILER; + +namespace { + +struct CopyEdge { + CgRegister Dst; + CgRegister Src; +}; + +class PhiEliminationImpl { +public: + void runOnCgFunction(CgFunction &MF) { + this->MF = &MF; + MRI = &MF.getRegInfo(); + TII = &MF.getTargetInstrInfo(); + + std::vector Blocks; + Blocks.reserve(MF.size()); + for (CgBasicBlock *BB : MF) { + Blocks.push_back(BB); + } + + for (CgBasicBlock *BB : Blocks) { + eliminateBlockPhis(*BB); + } + } + +private: + void eliminateBlockPhis(CgBasicBlock &BB) { + std::vector Phis; + for (CgInstruction &MI : BB) { + if (!MI.isPHI()) { + break; + } + Phis.push_back(&MI); + } + if (Phis.empty()) { + return; + } + + std::unordered_map> EdgeCopies; + EdgeCopies.reserve(BB.pred_size()); + + for (CgInstruction *Phi : Phis) { + ZEN_ASSERT(Phi->getNumOperands() >= 3 && + (Phi->getNumOperands() % 2) == 1 && "invalid lowered PHI"); + + const CgRegister DstReg = Phi->getOperand(0).getReg(); + for (unsigned OpIdx = 1; OpIdx < Phi->getNumOperands(); OpIdx += 2) { + CgOperand &SrcOp = Phi->getOperand(OpIdx); + CgOperand &PredOp = Phi->getOperand(OpIdx + 1); + ZEN_ASSERT(SrcOp.isReg() && PredOp.isMBB() && + "invalid lowered PHI incoming"); + EdgeCopies[PredOp.getMBB()].push_back(CopyEdge{DstReg, SrcOp.getReg()}); + } + } + + for (auto &[Pred, Copies] : EdgeCopies) { + eraseIdentityCopies(Copies); + if (Copies.empty()) { + continue; + } + + if (Pred->succ_size() == 1) { + emitParallelCopies(*Pred, Pred->getFirstTerminator(), Copies); + continue; + } + + CgBasicBlock *SplitBB = createSplitEdgeBlock(*Pred, BB); + emitParallelCopies(*SplitBB, SplitBB->end(), Copies); + addUnconditionalBranch(*SplitBB, BB); + } + + for (CgInstruction *Phi : Phis) { + Phi->eraseFromParent(); + } + } + + static void eraseIdentityCopies(std::vector &Copies) { + Copies.erase(std::remove_if( + Copies.begin(), Copies.end(), + [](const CopyEdge &Copy) { return Copy.Dst == Copy.Src; }), + Copies.end()); + } + + bool rewriteExplicitBranchTarget(CgBasicBlock &Pred, CgBasicBlock &OldSucc, + CgBasicBlock &NewSucc) { + bool Rewritten = false; + for (CgInstruction &MI : Pred.terminators()) { + if (!MI.isBranch()) { + continue; + } + for (CgOperand &MO : MI.operands()) { + if (!MO.isMBB() || MO.getMBB() != &OldSucc) { + continue; + } + MO.setMBB(&NewSucc); + Rewritten = true; + } + } + return Rewritten; + } + + CgBasicBlock *createSplitEdgeBlock(CgBasicBlock &Pred, CgBasicBlock &Succ) { + CgBasicBlock *SplitBB = MF->createCgBasicBlock(); + + const bool Rewritten = rewriteExplicitBranchTarget(Pred, Succ, *SplitBB); + if (Rewritten) { + MF->appendCgBasicBlock(SplitBB); + } else { + ZEN_ASSERT(Pred.isLayoutSuccessor(&Succ) && + "critical edge without explicit branch must be fallthrough"); + MF->insertCgBasicBlockAfter(&Pred, SplitBB); + } + + Pred.replaceSuccessor(&Succ, SplitBB); + SplitBB->addSuccessorWithoutProb(&Succ); + return SplitBB; + } + + const llvm::TargetRegisterClass *getCopyRegClass(const CopyEdge &Copy) const { + if (Copy.Dst.isVirtual()) { + return MRI->getRegClass(Copy.Dst); + } + if (Copy.Src.isVirtual()) { + return MRI->getRegClass(Copy.Src); + } + ZEN_ASSERT(false && "expected at least one virtual register in PHI copy"); + return nullptr; + } + + void addCopy(CgBasicBlock &BB, CgBasicBlock::iterator InsertPt, + CgRegister Dst, CgRegister Src) { + std::vector Operands = { + CgOperand::createRegOperand(Dst, true), + CgOperand::createRegOperand(Src, false), + }; + llvm::MutableArrayRef OperandRef(Operands); + MF->createCgInstruction(BB, InsertPt, TII->get(llvm::TargetOpcode::COPY), + OperandRef); + } + + void emitParallelCopies(CgBasicBlock &BB, CgBasicBlock::iterator InsertPt, + std::vector Copies) { + while (!Copies.empty()) { + bool Progress = false; + for (size_t Index = 0; Index < Copies.size(); ++Index) { + const CopyEdge &Copy = Copies[Index]; + const bool DstStillUsedAsSource = + std::any_of(Copies.begin(), Copies.end(), + [&](const CopyEdge &It) { return It.Src == Copy.Dst; }); + if (DstStillUsedAsSource) { + continue; + } + addCopy(BB, InsertPt, Copy.Dst, Copy.Src); + Copies.erase(Copies.begin() + Index); + Progress = true; + break; + } + if (Progress) { + continue; + } + + CopyEdge &CycleCopy = Copies.front(); + const llvm::TargetRegisterClass *RegClass = getCopyRegClass(CycleCopy); + CgRegister TempReg = MRI->createVirtualRegister(RegClass); + addCopy(BB, InsertPt, TempReg, CycleCopy.Dst); + for (CopyEdge &Copy : Copies) { + if (Copy.Src == CycleCopy.Dst) { + Copy.Src = TempReg; + } + } + } + } + + void addUnconditionalBranch(CgBasicBlock &BB, CgBasicBlock &TargetBB) { + std::vector Operands = { + CgOperand::createMBB(&TargetBB), + }; + llvm::MutableArrayRef OperandRef(Operands); + MF->createCgInstruction(BB, TII->get(X86::JMP_1), OperandRef); + } + + CgFunction *MF = nullptr; + CgRegisterInfo *MRI = nullptr; + const llvm::TargetInstrInfo *TII = nullptr; +}; + +} // namespace + +void CgPhiElimination::runOnCgFunction(CgFunction &MF) { + PhiEliminationImpl Impl; + Impl.runOnCgFunction(MF); +} diff --git a/src/compiler/cgir/pass/phi_elimination.h b/src/compiler/cgir/pass/phi_elimination.h new file mode 100644 index 000000000..65ab9acf5 --- /dev/null +++ b/src/compiler/cgir/pass/phi_elimination.h @@ -0,0 +1,17 @@ +// Copyright (C) 2025 the DTVM authors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#pragma once + +#include "compiler/common/common_defs.h" + +namespace COMPILER { + +class CgFunction; + +class CgPhiElimination : public NonCopyable { +public: + void runOnCgFunction(CgFunction &MF); +}; + +} // namespace COMPILER diff --git a/src/compiler/compiler.cpp b/src/compiler/compiler.cpp index 5fe2d2ac1..f14962d63 100644 --- a/src/compiler/compiler.cpp +++ b/src/compiler/compiler.cpp @@ -7,6 +7,7 @@ #include "compiler/cgir/pass/dead_cg_instruction_elim.h" #include "compiler/cgir/pass/expand_post_ra_pseudos.h" #include "compiler/cgir/pass/fast_ra.h" +#include "compiler/cgir/pass/phi_elimination.h" #include "compiler/cgir/pass/prolog_epilog_inserter.h" #include "compiler/cgir/pass/reg_alloc_basic.h" #include "compiler/cgir/pass/reg_alloc_greedy.h" @@ -75,6 +76,8 @@ void JITCompilerBase::compileMIRToCgIR(MModule &MMod, MFunction &MFunc, // TODO: refactor to pass X86CgLowering CgLowering(MF); X86CgPeephole CgPeephole(MF); + CgPhiElimination PhiElimination; + PhiElimination.runOnCgFunction(MF); uint32_t MFuncIdx = MFunc.getFuncIdx(); diff --git a/src/compiler/evm_frontend/evm_analyzer.h b/src/compiler/evm_frontend/evm_analyzer.h index bf70d5224..9820c647a 100644 --- a/src/compiler/evm_frontend/evm_analyzer.h +++ b/src/compiler/evm_frontend/evm_analyzer.h @@ -4,12 +4,17 @@ #ifndef EVM_FRONTEND_EVM_ANALYZER_H #define EVM_FRONTEND_EVM_ANALYZER_H -#include "compiler/common/common_defs.h" +#include "common/defines.h" #include "evm/evm.h" #include "evmc/evmc.h" #include "evmc/instructions.h" #include +#include +#include +#include +#include +#include namespace COMPILER { @@ -121,41 +126,370 @@ static constexpr size_t MAX_DUP_FEEDBACK_PATTERN = 64; class EVMAnalyzer { using Byte = zen::common::Byte; - using Bytes = zen::common::Bytes; public: EVMAnalyzer(evmc_revision Rev = zen::evm::DEFAULT_REVISION) : Revision(Rev) {} struct BlockInfo { uint64_t EntryPC = 0; + uint64_t BodyStartPC = 0; + uint64_t BodyEndPC = 0; int32_t MaxStackHeight = 0; int32_t MinStackHeight = 0; int32_t MinPopHeight = 0; int32_t StackHeightDiff = 0; + int32_t EntryStackDepth = 0; + int32_t ResolvedEntryStackDepth = -1; + int32_t ResolvedExitStackDepth = -1; + int32_t FullEntryStateDepth = -1; + int32_t HiddenLiveInPrefixDepth = 0; + bool HasInconsistentEntryDepth = false; + bool IsEntryStateCompatible = false; + bool HasHiddenLiveInPrefix = false; + bool RequiresEntryMergeState = false; + bool HasDeferredEntryMerge = false; + bool IsDynamicJumpTargetCandidate = false; + bool HasCompatibleDynamicJumpTargetShape = false; bool IsJumpDest = false; bool HasUndefinedInstr = false; + bool HasDynamicJump = false; + bool HasConditionalJump = false; + bool HasConstantJump = false; + bool CanLiftStack = false; + uint64_t ConstantJumpTargetPC = 0; + uint64_t DynamicJumpTargetRegionEntryPC = 0; + std::vector DynamicJumpTargetRegions; uint32_t RAExpensiveCount = 0; + std::vector Successors; + std::vector Predecessors; BlockInfo() = default; - BlockInfo(uint64_t PC) : EntryPC(PC) {} + BlockInfo(uint64_t PC, uint64_t StartPC = 0, bool JumpDest = false) + : EntryPC(PC), BodyStartPC(StartPC), BodyEndPC(StartPC), + IsJumpDest(JumpDest) {} }; const std::map &getBlockInfos() const { return BlockInfos; } - /// Return the JIT suitability result computed during the last analyze() call. + struct DynamicJumpRegionInfo { + uint64_t RegionEntryPC = 0; + std::vector SourceBlocks; + std::vector TargetBlocks; + int32_t UniformEntryDepth = -1; + int32_t FullEntryStateDepth = -1; + int32_t HiddenLiveInPrefixDepth = 0; + bool RequiresEntryMergeState = false; + bool HasUniformEntryDepth = false; + bool HasCompatibleTargetShape = false; + uint32_t ShapeClassId = 0; + }; + + const std::map & + getDynamicJumpRegions() const { + return DynamicJumpRegions; + } + + const DynamicJumpRegionInfo * + getDynamicJumpRegionInfo(uint64_t RegionEntryPC) const { + auto It = DynamicJumpRegions.find(RegionEntryPC); + if (It == DynamicJumpRegions.end()) { + return nullptr; + } + return &It->second; + } + + std::vector + getCompatibleDynamicJumpShapeClassIdsForBlock(uint64_t BlockPC) const { + std::vector ShapeClassIds; + auto It = BlockInfos.find(BlockPC); + if (It == BlockInfos.end()) { + return ShapeClassIds; + } + + for (uint64_t RegionEntryPC : It->second.DynamicJumpTargetRegions) { + const auto *RegionInfo = getDynamicJumpRegionInfo(RegionEntryPC); + if (!RegionInfo || !RegionInfo->HasCompatibleTargetShape || + RegionInfo->ShapeClassId == 0) { + continue; + } + if (std::find(ShapeClassIds.begin(), ShapeClassIds.end(), + RegionInfo->ShapeClassId) == ShapeClassIds.end()) { + ShapeClassIds.push_back(RegionInfo->ShapeClassId); + } + } + return ShapeClassIds; + } + + uint32_t + getUniqueCompatibleDynamicJumpShapeClassForBlock(uint64_t BlockPC) const { + const std::vector ShapeClassIds = + getCompatibleDynamicJumpShapeClassIdsForBlock(BlockPC); + return ShapeClassIds.size() == 1 ? ShapeClassIds.front() : 0; + } + + uint32_t + getOutgoingCompatibleDynamicJumpShapeClassForBlock(uint64_t BlockPC) const { + auto It = BlockInfos.find(BlockPC); + if (It == BlockInfos.end()) { + return 0; + } + if (!It->second.HasDynamicJump || + It->second.DynamicJumpTargetRegionEntryPC == 0) { + return 0; + } + + const auto *RegionInfo = + getDynamicJumpRegionInfo(It->second.DynamicJumpTargetRegionEntryPC); + if (!RegionInfo || !RegionInfo->HasCompatibleTargetShape) { + return 0; + } + return RegionInfo->ShapeClassId; + } + + bool blockHasCompatibleDynamicJumpShapeClass(uint64_t BlockPC, + uint32_t ShapeClassId) const { + if (ShapeClassId == 0) { + return false; + } + const std::vector ShapeClassIds = + getCompatibleDynamicJumpShapeClassIdsForBlock(BlockPC); + return std::find(ShapeClassIds.begin(), ShapeClassIds.end(), + ShapeClassId) != ShapeClassIds.end(); + } + + bool blocksShareCompatibleDynamicJumpShapeClass(uint64_t BlockPC, + uint64_t OtherBlockPC) const { + const std::vector ShapeClassIds = + getCompatibleDynamicJumpShapeClassIdsForBlock(BlockPC); + const std::vector OtherShapeClassIds = + getCompatibleDynamicJumpShapeClassIdsForBlock(OtherBlockPC); + for (uint32_t ShapeClassId : ShapeClassIds) { + if (std::find(OtherShapeClassIds.begin(), OtherShapeClassIds.end(), + ShapeClassId) != OtherShapeClassIds.end()) { + return true; + } + } + + const uint32_t OutgoingShapeClassId = + getOutgoingCompatibleDynamicJumpShapeClassForBlock(BlockPC); + if (OutgoingShapeClassId != 0 && + (blockHasCompatibleDynamicJumpShapeClass(OtherBlockPC, + OutgoingShapeClassId) || + OutgoingShapeClassId == + getOutgoingCompatibleDynamicJumpShapeClassForBlock( + OtherBlockPC))) { + return true; + } + + const uint32_t OtherOutgoingShapeClassId = + getOutgoingCompatibleDynamicJumpShapeClassForBlock(OtherBlockPC); + return OtherOutgoingShapeClassId != 0 && + blockHasCompatibleDynamicJumpShapeClass(BlockPC, + OtherOutgoingShapeClassId); + } + + std::vector + getDynamicJumpSourceBlocksForBlock(uint64_t BlockPC) const { + auto It = BlockInfos.find(BlockPC); + if (It == BlockInfos.end()) { + return {}; + } + return collectDynamicJumpSourceBlocksForInfo(It->second); + } + + std::vector + getPotentialEntryPredecessorsForBlock(uint64_t BlockPC) const { + auto It = BlockInfos.find(BlockPC); + if (It == BlockInfos.end()) { + return {}; + } + + std::vector PredBlockPCs(It->second.Predecessors.begin(), + It->second.Predecessors.end()); + for (uint64_t PredBlockPC : + collectDynamicJumpSourceBlocksForInfo(It->second)) { + appendUniqueBlockPC(PredBlockPCs, PredBlockPC); + } + return PredBlockPCs; + } + + std::vector + getCompatibleDynamicJumpTargetBlocksForSourceBlock(uint64_t BlockPC) const { + auto It = BlockInfos.find(BlockPC); + if (It == BlockInfos.end()) { + return {}; + } + uint32_t OutgoingShapeClassId = + getOutgoingCompatibleDynamicJumpShapeClassForBlock(BlockPC); + if (OutgoingShapeClassId == 0 && It->second.HasDynamicJump) { + OutgoingShapeClassId = + getUniqueCompatibleDynamicJumpShapeClassForBlock(BlockPC); + } + if (OutgoingShapeClassId != 0) { + std::vector TargetBlockPCs; + for (const auto &[EntryPC, Info] : BlockInfos) { + if (!Info.HasCompatibleDynamicJumpTargetShape) { + continue; + } + if (blockHasCompatibleDynamicJumpShapeClass(EntryPC, + OutgoingShapeClassId)) { + appendUniqueBlockPC(TargetBlockPCs, EntryPC); + } + } + return TargetBlockPCs; + } + if (!It->second.HasDynamicJump || + It->second.DynamicJumpTargetRegionEntryPC == 0) { + return {}; + } + + const auto *RegionInfo = + getDynamicJumpRegionInfo(It->second.DynamicJumpTargetRegionEntryPC); + if (!RegionInfo || !RegionInfo->HasCompatibleTargetShape) { + return {}; + } + return RegionInfo->TargetBlocks; + } + + bool hasDeferredLiftedEntryMerge(uint64_t BlockPC) const { + auto It = BlockInfos.find(BlockPC); + return It != BlockInfos.end() && It->second.CanLiftStack && + It->second.HasDeferredEntryMerge; + } + + bool canTransferLiftedEntryStateWithoutRuntimeMaterialization( + uint64_t BlockPC) const { + auto It = BlockInfos.find(BlockPC); + return It != BlockInfos.end() && It->second.CanLiftStack; + } + + bool canTransferCompatibleDynamicJumpTargetsWithoutRuntimeMaterialization( + uint64_t BlockPC) const { + const std::vector TargetBlockPCs = + getCompatibleDynamicJumpTargetBlocksForSourceBlock(BlockPC); + return !TargetBlockPCs.empty() && + std::all_of( + TargetBlockPCs.begin(), TargetBlockPCs.end(), + [this](uint64_t TargetBlockPC) { + return canTransferLiftedEntryStateWithoutRuntimeMaterialization( + TargetBlockPC); + }); + } + const JITSuitabilityResult &getJITSuitability() const { return JITResult; } + bool hasCanonicalJumpDest(uint64_t PC) const { + return JumpDestCanonicalPCs.count(PC) != 0; + } + + uint64_t getCanonicalJumpDestPC(uint64_t PC) const { + auto It = JumpDestCanonicalPCs.find(PC); + return It == JumpDestCanonicalPCs.end() ? PC : It->second; + } + + bool hasUnknownDynamicJumpTargets() const { return HasUnknownDynamicJump; } + + bool analyzeSuitabilityOnly(const uint8_t *Bytecode, size_t BytecodeSize) { + resetAnalysisState(); + analyzeSuitability(Bytecode, BytecodeSize); + return true; + } + bool analyze(const uint8_t *Bytecode, size_t BytecodeSize) { + resetAnalysisState(); + analyzeSuitability(Bytecode, BytecodeSize); + buildJumpDestRuns(Bytecode, BytecodeSize); + buildBlocks(Bytecode, BytecodeSize); + linkPredecessors(); + resolveEntryDepths(); + markDynamicJumpTargetCandidates(); + resolveDynamicJumpTargetEntryDepths(); + finalizeEntryShapeMetadata(); + finalizeLiftability(); + return true; + } + +private: + void resetAnalysisState() { BlockInfos.clear(); + DynamicJumpRegions.clear(); + JumpDestCanonicalPCs.clear(); + EntryBlockPC = 0; + HasUnknownDynamicJump = false; + } + + struct AbstractValue { + bool KnownConst = false; + bool FitsU64 = false; + uint64_t Low = 0; + + static AbstractValue unknown() { return {}; } + + static AbstractValue constFromPush(const uint8_t *Bytecode, + size_t BytecodeSize, size_t Start, + size_t Size) { + AbstractValue V; + V.KnownConst = true; + V.FitsU64 = true; + V.Low = 0; + if (Size == 0) { + return V; + } + + const size_t Available = + Start < BytecodeSize ? (BytecodeSize - Start) : 0; + const size_t ReadCount = std::min(Size, Available); + auto readPushByte = [&](size_t Index) -> uint8_t { + if (Index >= ReadCount) { + return 0; + } + return Bytecode[Start + Index]; + }; + + size_t ValueStart = 0; + if (Size > sizeof(uint64_t)) { + for (size_t I = 0; I < Size - sizeof(uint64_t); ++I) { + if (readPushByte(I) != 0) { + V.FitsU64 = false; + break; + } + } + ValueStart = Size - sizeof(uint64_t); + } + for (size_t I = ValueStart; I < Size; ++I) { + V.Low = (V.Low << 8) | static_cast(readPushByte(I)); + } + return V; + } + }; + + static bool isBlockTerminator(evmc_opcode Opcode) { + switch (Opcode) { + case OP_JUMP: + case OP_STOP: + case OP_RETURN: + case OP_INVALID: + case OP_REVERT: + case OP_SELFDESTRUCT: + return true; + default: + return false; + } + } + + static size_t immediateSize(evmc_opcode Opcode) { + if (Opcode >= OP_PUSH0 && Opcode <= OP_PUSH32) { + return static_cast(Opcode - OP_PUSH0); + } + return 0; + } + + void analyzeSuitability(const uint8_t *Bytecode, size_t BytecodeSize) { JITResult = JITSuitabilityResult(); JITResult.BytecodeSize = BytecodeSize; - const uint8_t *Ip = Bytecode; - const uint8_t *IpEnd = Bytecode + BytecodeSize; - - // Get instruction tables based on revision const auto *InstructionMetrics = evmc_get_instruction_metrics_table(Revision); const auto *InstructionNames = evmc_get_instruction_names_table(Revision); @@ -168,157 +502,847 @@ class EVMAnalyzer { evmc_get_instruction_names_table(zen::evm::DEFAULT_REVISION); } - // Initialize block info for the first block - BlockInfo CurInfo(0); - - // JIT suitability tracking state size_t CurConsecutiveExpensive = 0; + size_t CurBlockExpensiveCount = 0; bool PrevWasDup = false; - while (Ip < IpEnd) { - evmc_opcode Opcode = static_cast(*Ip); + size_t PCIndex = 0; + while (PCIndex < BytecodeSize) { + evmc_opcode Opcode = static_cast(Bytecode[PCIndex]); uint8_t OpcodeU8 = static_cast(Opcode); - ptrdiff_t Diff = Ip - Bytecode; - PC = static_cast(Diff >= 0 ? Diff : 0); - - Ip++; - // --- JIT suitability: accumulate MIR estimate --- JITResult.MirEstimate += MIR_OPCODE_WEIGHT[OpcodeU8]; - // --- JIT suitability: RA-expensive pattern tracking --- if (isRAExpensiveOpcode(OpcodeU8)) { JITResult.RAExpensiveCount++; - CurInfo.RAExpensiveCount++; + CurBlockExpensiveCount++; CurConsecutiveExpensive++; - // DUP feedback: previous opcode was DUP, now RA-expensive if (PrevWasDup) { JITResult.DupFeedbackPatternCount++; } PrevWasDup = false; } else if (isDupOrSwapOpcode(OpcodeU8)) { - // DUP/SWAP are transparent — don't break consecutive run PrevWasDup = isDupOpcode(OpcodeU8); } else { - // Any other opcode breaks the consecutive run JITResult.MaxConsecutiveExpensive = std::max( JITResult.MaxConsecutiveExpensive, CurConsecutiveExpensive); CurConsecutiveExpensive = 0; PrevWasDup = false; } - // Check if opcode is undefined for current revision + bool IsBlockBoundary = (Opcode == OP_JUMPI || Opcode == OP_JUMPDEST || + isBlockTerminator(Opcode)); + if (IsBlockBoundary) { + JITResult.MaxBlockExpensiveCount = + std::max(JITResult.MaxBlockExpensiveCount, CurBlockExpensiveCount); + CurBlockExpensiveCount = 0; + JITResult.MaxConsecutiveExpensive = std::max( + JITResult.MaxConsecutiveExpensive, CurConsecutiveExpensive); + CurConsecutiveExpensive = 0; + PrevWasDup = false; + } + + size_t PushBytes = immediateSize(Opcode); + PCIndex += 1 + PushBytes; + + (void)InstructionMetrics; + (void)InstructionNames; + } + + JITResult.MaxConsecutiveExpensive = + std::max(JITResult.MaxConsecutiveExpensive, CurConsecutiveExpensive); + JITResult.MaxBlockExpensiveCount = + std::max(JITResult.MaxBlockExpensiveCount, CurBlockExpensiveCount); + + JITResult.ShouldFallback = + BytecodeSize > MAX_JIT_BYTECODE_SIZE || + JITResult.MirEstimate > MAX_JIT_MIR_ESTIMATE || + JITResult.MaxConsecutiveExpensive > MAX_CONSECUTIVE_RA_EXPENSIVE || + JITResult.MaxBlockExpensiveCount > MAX_BLOCK_RA_EXPENSIVE || + JITResult.DupFeedbackPatternCount > MAX_DUP_FEEDBACK_PATTERN; + } + + void buildJumpDestRuns(const uint8_t *Bytecode, size_t BytecodeSize) { + size_t PCIndex = 0; + while (PCIndex < BytecodeSize) { + evmc_opcode Opcode = static_cast(Bytecode[PCIndex]); + if (Opcode == OP_JUMPDEST) { + size_t RunStart = PCIndex; + size_t RunEnd = PCIndex; + while (RunEnd + 1 < BytecodeSize && + static_cast(Bytecode[RunEnd + 1]) == OP_JUMPDEST) { + ++RunEnd; + } + for (size_t PC = RunStart; PC <= RunEnd; ++PC) { + JumpDestCanonicalPCs[static_cast(PC)] = + static_cast(RunEnd); + } + PCIndex = RunEnd + 1; + continue; + } + PCIndex += 1 + immediateSize(Opcode); + } + } + + void ensureAbstractDepth(std::vector &Stack, + size_t &EntryDepth, size_t RequiredDepth) { + if (Stack.size() >= RequiredDepth) { + return; + } + size_t Deficit = RequiredDepth - Stack.size(); + Stack.insert(Stack.begin(), Deficit, AbstractValue::unknown()); + EntryDepth += Deficit; + } + + void analyzeBlockBody(BlockInfo &Info, const uint8_t *Bytecode, + size_t BytecodeSize, size_t &ScanPC, + uint64_t &NextEntryPC, size_t &NextBodyStartPC, + bool &HasNextBlock) { + const auto *InstructionMetrics = + evmc_get_instruction_metrics_table(Revision); + const auto *InstructionNames = evmc_get_instruction_names_table(Revision); + if (!InstructionMetrics) { + InstructionMetrics = + evmc_get_instruction_metrics_table(zen::evm::DEFAULT_REVISION); + } + if (!InstructionNames) { + InstructionNames = + evmc_get_instruction_names_table(zen::evm::DEFAULT_REVISION); + } + + std::vector Stack; + size_t EntryDepth = 0; + Info.MaxStackHeight = 0; + Info.MinStackHeight = 0; + Info.MinPopHeight = 0; + Info.StackHeightDiff = 0; + Info.EntryStackDepth = 0; + Info.BodyEndPC = Info.BodyStartPC; + + auto updateHeights = [&]() { + int32_t RelativeHeight = + static_cast(Stack.size()) - static_cast(EntryDepth); + Info.StackHeightDiff = RelativeHeight; + Info.MaxStackHeight = std::max(Info.MaxStackHeight, RelativeHeight); + Info.MinStackHeight = std::min(Info.MinStackHeight, RelativeHeight); + Info.MinPopHeight = + std::min(Info.MinPopHeight, -static_cast(EntryDepth)); + }; + + HasNextBlock = false; + NextEntryPC = 0; + NextBodyStartPC = BytecodeSize; + + while (ScanPC < BytecodeSize) { + evmc_opcode Opcode = static_cast(Bytecode[ScanPC]); + + if (Opcode == OP_JUMPDEST) { + uint64_t CanonicalPC = + getCanonicalJumpDestPC(static_cast(ScanPC)); + Info.Successors.push_back(CanonicalPC); + NextEntryPC = CanonicalPC; + NextBodyStartPC = static_cast(CanonicalPC) + 1; + HasNextBlock = true; + Info.BodyEndPC = static_cast(ScanPC); + break; + } + bool IsUndefined = (InstructionNames[Opcode] == nullptr); if (IsUndefined) { - CurInfo.HasUndefinedInstr = true; + Info.HasUndefinedInstr = true; #ifdef ZEN_ENABLE_JIT_FALLBACK_TEST - // Reset undefined instruction flag in fallback test - CurInfo.HasUndefinedInstr = false; + Info.HasUndefinedInstr = false; #endif + ++ScanPC; + Info.BodyEndPC = static_cast(ScanPC); + skipDeadCode(Bytecode, BytecodeSize, ScanPC, NextEntryPC, + NextBodyStartPC, HasNextBlock); + break; + } + + uint8_t OpcodeU8 = static_cast(Opcode); + if (isRAExpensiveOpcode(OpcodeU8)) { + Info.RAExpensiveCount++; + } + + ++ScanPC; + size_t PushBytes = immediateSize(Opcode); + + if (Opcode == OP_JUMP) { + ensureAbstractDepth(Stack, EntryDepth, 1); + AbstractValue Dest = Stack.back(); + Stack.pop_back(); + updateHeights(); + + if (Dest.KnownConst && Dest.FitsU64 && hasCanonicalJumpDest(Dest.Low)) { + Info.HasConstantJump = true; + Info.ConstantJumpTargetPC = getCanonicalJumpDestPC(Dest.Low); + Info.Successors.push_back(Info.ConstantJumpTargetPC); + } else { + Info.HasDynamicJump = true; + HasUnknownDynamicJump = true; + } + Info.BodyEndPC = static_cast(ScanPC); + skipDeadCode(Bytecode, BytecodeSize, ScanPC, NextEntryPC, + NextBodyStartPC, HasNextBlock); + if (Info.HasDynamicJump && HasNextBlock) { + Info.DynamicJumpTargetRegionEntryPC = NextEntryPC; + } + break; + } + + if (Opcode == OP_JUMPI) { + ensureAbstractDepth(Stack, EntryDepth, 2); + AbstractValue Dest = Stack.back(); + Stack.pop_back(); + Stack.pop_back(); + updateHeights(); + + uint64_t FallthroughEntryPC = static_cast(ScanPC); + size_t FallthroughBodyStartPC = ScanPC; + if (ScanPC < BytecodeSize && + static_cast(Bytecode[ScanPC]) == OP_JUMPDEST) { + FallthroughEntryPC = getCanonicalJumpDestPC(FallthroughEntryPC); + FallthroughBodyStartPC = static_cast(FallthroughEntryPC) + 1; + } + + Info.HasConditionalJump = true; + Info.Successors.push_back(FallthroughEntryPC); + if (Dest.KnownConst && Dest.FitsU64 && hasCanonicalJumpDest(Dest.Low)) { + Info.HasConstantJump = true; + Info.ConstantJumpTargetPC = getCanonicalJumpDestPC(Dest.Low); + if (Info.ConstantJumpTargetPC != FallthroughEntryPC) { + Info.Successors.push_back(Info.ConstantJumpTargetPC); + } + } else if (!Dest.KnownConst || !Dest.FitsU64) { + Info.HasDynamicJump = true; + HasUnknownDynamicJump = true; + Info.DynamicJumpTargetRegionEntryPC = FallthroughEntryPC; + } + NextEntryPC = FallthroughEntryPC; + NextBodyStartPC = FallthroughBodyStartPC; + HasNextBlock = true; + Info.BodyEndPC = static_cast(ScanPC); + break; + } + + if (isBlockTerminator(Opcode)) { + const auto &Metrics = InstructionMetrics[Opcode]; + int PopCount = Metrics.stack_height_required; + int PushCount = PopCount + Metrics.stack_height_change; + ensureAbstractDepth(Stack, EntryDepth, static_cast(PopCount)); + for (int I = 0; I < PopCount; ++I) { + Stack.pop_back(); + } + for (int I = 0; I < PushCount; ++I) { + Stack.push_back(AbstractValue::unknown()); + } + updateHeights(); + Info.BodyEndPC = static_cast(ScanPC); + skipDeadCode(Bytecode, BytecodeSize, ScanPC, NextEntryPC, + NextBodyStartPC, HasNextBlock); + break; + } + + if (Opcode >= OP_DUP1 && Opcode <= OP_DUP16) { + size_t RequiredDepth = static_cast(Opcode - OP_DUP1 + 1); + ensureAbstractDepth(Stack, EntryDepth, RequiredDepth); + Stack.push_back(Stack[Stack.size() - RequiredDepth]); + updateHeights(); + } else if (Opcode >= OP_SWAP1 && Opcode <= OP_SWAP16) { + size_t RequiredDepth = static_cast(Opcode - OP_SWAP1 + 2); + ensureAbstractDepth(Stack, EntryDepth, RequiredDepth); + std::swap(Stack.back(), Stack[Stack.size() - RequiredDepth]); + updateHeights(); + } else if (Opcode >= OP_PUSH0 && Opcode <= OP_PUSH32) { + Stack.push_back(AbstractValue::constFromPush(Bytecode, BytecodeSize, + ScanPC, PushBytes)); + ScanPC += PushBytes; + updateHeights(); + } else { + const auto &Metrics = InstructionMetrics[Opcode]; + int PopCount = Metrics.stack_height_required; + int PushCount = PopCount + Metrics.stack_height_change; + ensureAbstractDepth(Stack, EntryDepth, static_cast(PopCount)); + for (int I = 0; I < PopCount; ++I) { + Stack.pop_back(); + } + for (int I = 0; I < PushCount; ++I) { + Stack.push_back(AbstractValue::unknown()); + } + updateHeights(); + } + } + + if (ScanPC >= BytecodeSize) { + Info.BodyEndPC = static_cast(BytecodeSize); + } + + Info.EntryStackDepth = static_cast(EntryDepth); + Info.MinStackHeight = std::min(Info.MinStackHeight, -Info.EntryStackDepth); + Info.MinPopHeight = std::min(Info.MinPopHeight, -Info.EntryStackDepth); + Info.StackHeightDiff = + static_cast(Stack.size()) - static_cast(EntryDepth); + } + + void skipDeadCode(const uint8_t *Bytecode, size_t BytecodeSize, + size_t &ScanPC, uint64_t &NextEntryPC, + size_t &NextBodyStartPC, bool &HasNextBlock) { + while (ScanPC < BytecodeSize) { + evmc_opcode NextOp = static_cast(Bytecode[ScanPC]); + if (NextOp == OP_JUMPDEST) { + uint64_t CanonicalPC = + getCanonicalJumpDestPC(static_cast(ScanPC)); + NextEntryPC = CanonicalPC; + NextBodyStartPC = static_cast(CanonicalPC) + 1; + HasNextBlock = true; + return; } + ScanPC += 1 + immediateSize(NextOp); + } + HasNextBlock = false; + } - // Get stack metrics from the instruction metrics table - const auto &Metrics = InstructionMetrics[Opcode]; - // stack_height_required equals PopCount - int PopCount = Metrics.stack_height_required; - // PushCount = PopCount + stack_height_change - int PushCount = PopCount + Metrics.stack_height_change; + void buildBlocks(const uint8_t *Bytecode, size_t BytecodeSize) { + if (BytecodeSize == 0) { + BlockInfos.emplace(0, BlockInfo(0, 0, false)); + EntryBlockPC = 0; + return; + } - // Handle PUSH instructions - need to skip the immediate bytes - if (Opcode >= OP_PUSH1 && Opcode <= OP_PUSH32) { - uint8_t PushBytes = Opcode - OP_PUSH0; - Ip += PushBytes; + size_t BodyStartPC = 0; + bool StartsWithJumpDest = + static_cast(Bytecode[0]) == OP_JUMPDEST; + bool IsJumpDestBlock = false; + if (StartsWithJumpDest) { + EntryBlockPC = getCanonicalJumpDestPC(0); + BodyStartPC = static_cast(EntryBlockPC) + 1; + IsJumpDestBlock = true; + } else { + EntryBlockPC = 0; + } + + uint64_t CurEntryPC = EntryBlockPC; + while (true) { + BlockInfo Info(CurEntryPC, BodyStartPC, IsJumpDestBlock); + size_t ScanPC = BodyStartPC; + uint64_t NextEntryPC = 0; + size_t NextBodyStartPC = BytecodeSize; + bool HasNextBlock = false; + analyzeBlockBody(Info, Bytecode, BytecodeSize, ScanPC, NextEntryPC, + NextBodyStartPC, HasNextBlock); + BlockInfos[CurEntryPC] = Info; + if (!HasNextBlock) { + break; + } + CurEntryPC = NextEntryPC; + BodyStartPC = NextBodyStartPC; + IsJumpDestBlock = hasCanonicalJumpDest(CurEntryPC) && + getCanonicalJumpDestPC(CurEntryPC) == CurEntryPC; + if (BodyStartPC > BytecodeSize) { + break; } + } + } - // Update stack height - CurInfo.StackHeightDiff -= PopCount; - if (CurInfo.StackHeightDiff < CurInfo.MinStackHeight) { - CurInfo.MinStackHeight = CurInfo.StackHeightDiff; + void linkPredecessors() { + for (auto &[EntryPC, Info] : BlockInfos) { + (void)EntryPC; + for (uint64_t Succ : Info.Successors) { + auto It = BlockInfos.find(Succ); + if (It == BlockInfos.end()) { + continue; + } + auto &Preds = It->second.Predecessors; + if (std::find(Preds.begin(), Preds.end(), Info.EntryPC) == + Preds.end()) { + Preds.push_back(Info.EntryPC); + } } - if (!(Opcode >= OP_SWAP1 && Opcode <= OP_SWAP16) && - !(Opcode >= OP_DUP1 && Opcode <= OP_DUP16)) { - CurInfo.MinPopHeight = - std::min(CurInfo.StackHeightDiff, CurInfo.MinPopHeight); + } + } + + void invalidateReachableEntryDepths(uint64_t EntryPC) { + std::queue InvalidateWorkList; + std::map InvalidateVisited; + InvalidateWorkList.push(EntryPC); + InvalidateVisited[EntryPC] = true; + + while (!InvalidateWorkList.empty()) { + uint64_t InvalidPC = InvalidateWorkList.front(); + InvalidateWorkList.pop(); + auto InvalidIt = BlockInfos.find(InvalidPC); + if (InvalidIt == BlockInfos.end()) { + continue; } - CurInfo.StackHeightDiff += PushCount; - if (CurInfo.StackHeightDiff > CurInfo.MaxStackHeight) { - CurInfo.MaxStackHeight = CurInfo.StackHeightDiff; + auto &InvalidInfo = InvalidIt->second; + InvalidInfo.HasInconsistentEntryDepth = true; + InvalidInfo.ResolvedEntryStackDepth = -1; + InvalidInfo.ResolvedExitStackDepth = -1; + for (uint64_t NextSucc : InvalidInfo.Successors) { + if (InvalidateVisited.emplace(NextSucc, true).second) { + InvalidateWorkList.push(NextSucc); + } } + } + } - // Check if this is a block starting opcode - bool IsBlockStart = (Opcode == OP_JUMPDEST || Opcode == OP_JUMPI); - // Check if this is a block ending opcode - bool IsBlockEnd = (Opcode == OP_JUMP || Opcode == OP_RETURN || - Opcode == OP_STOP || Opcode == OP_INVALID || - Opcode == OP_REVERT || Opcode == OP_SELFDESTRUCT); + void resolveEntryDepths() { + auto EntryIt = BlockInfos.find(EntryBlockPC); + if (EntryIt == BlockInfos.end()) { + return; + } + + EntryIt->second.ResolvedEntryStackDepth = 0; + std::queue WorkList; + WorkList.push(EntryBlockPC); + propagateEntryDepths(WorkList); + } + + void propagateEntryDepths(std::queue &WorkList) { + while (!WorkList.empty()) { + uint64_t EntryPC = WorkList.front(); + WorkList.pop(); + auto &Info = BlockInfos[EntryPC]; + if (Info.ResolvedEntryStackDepth < 0) { + continue; + } + + int32_t ExitDepth = Info.ResolvedEntryStackDepth + Info.StackHeightDiff; + Info.ResolvedExitStackDepth = ExitDepth; - if (IsBlockStart) { - if (PC != CurInfo.EntryPC) { - // Finalize block: update max block RA-expensive count - JITResult.MaxBlockExpensiveCount = - std::max(JITResult.MaxBlockExpensiveCount, - static_cast(CurInfo.RAExpensiveCount)); - BlockInfos.emplace(CurInfo.EntryPC, CurInfo); + for (uint64_t Succ : Info.Successors) { + auto SuccIt = BlockInfos.find(Succ); + if (SuccIt == BlockInfos.end()) { + continue; } - // Create new block info - CurInfo = BlockInfo(PC); - if (Opcode == OP_JUMPDEST) { - CurInfo.IsJumpDest = true; + auto &SuccInfo = SuccIt->second; + if (SuccInfo.HasInconsistentEntryDepth) { + continue; } - // Block boundary also ends a consecutive run - JITResult.MaxConsecutiveExpensive = std::max( - JITResult.MaxConsecutiveExpensive, CurConsecutiveExpensive); - CurConsecutiveExpensive = 0; - } else if (IsBlockEnd) { - // Finalize block: update max block RA-expensive count - JITResult.MaxBlockExpensiveCount = - std::max(JITResult.MaxBlockExpensiveCount, - static_cast(CurInfo.RAExpensiveCount)); - // Save current block info - BlockInfos.emplace(CurInfo.EntryPC, CurInfo); - // Block boundary ends consecutive run - JITResult.MaxConsecutiveExpensive = std::max( - JITResult.MaxConsecutiveExpensive, CurConsecutiveExpensive); - CurConsecutiveExpensive = 0; - // Skip dead code - while (Ip < IpEnd) { - evmc_opcode NextOp = static_cast(*Ip); - if (NextOp == OP_JUMPDEST) { - break; - } - Ip++; - if (NextOp >= OP_PUSH0 && NextOp <= OP_PUSH32) { - uint8_t NumBytes = - static_cast(NextOp) - static_cast(OP_PUSH0); - Ip += NumBytes; + if (SuccInfo.ResolvedEntryStackDepth < 0) { + SuccInfo.ResolvedEntryStackDepth = ExitDepth; + WorkList.push(Succ); + } else if (SuccInfo.ResolvedEntryStackDepth != ExitDepth) { + invalidateReachableEntryDepths(Succ); + } + } + } + } + + std::vector collectReachableDynamicJumpRegions() const { + std::vector Regions; + for (const auto &[EntryPC, Info] : BlockInfos) { + (void)EntryPC; + if (!Info.HasDynamicJump || Info.ResolvedEntryStackDepth < 0 || + Info.DynamicJumpTargetRegionEntryPC == 0) { + continue; + } + if (std::find(Regions.begin(), Regions.end(), + Info.DynamicJumpTargetRegionEntryPC) == Regions.end()) { + Regions.push_back(Info.DynamicJumpTargetRegionEntryPC); + } + } + return Regions; + } + + static bool hasDynamicJumpRegion(const BlockInfo &Info, + uint64_t RegionEntryPC) { + return std::find(Info.DynamicJumpTargetRegions.begin(), + Info.DynamicJumpTargetRegions.end(), + RegionEntryPC) != Info.DynamicJumpTargetRegions.end(); + } + + static void addDynamicJumpRegion(BlockInfo &Info, uint64_t RegionEntryPC) { + if (!hasDynamicJumpRegion(Info, RegionEntryPC)) { + Info.DynamicJumpTargetRegions.push_back(RegionEntryPC); + } + } + + static void appendUniqueBlockPC(std::vector &BlockPCs, + uint64_t BlockPC) { + if (std::find(BlockPCs.begin(), BlockPCs.end(), BlockPC) == + BlockPCs.end()) { + BlockPCs.push_back(BlockPC); + } + } + + std::vector + collectDynamicJumpSourceBlocksForInfo(const BlockInfo &Info) const { + std::vector SourceBlockPCs; + if (Info.HasCompatibleDynamicJumpTargetShape) { + for (const auto &[EntryPC, RegionSourceInfo] : BlockInfos) { + if (!RegionSourceInfo.HasDynamicJump || + RegionSourceInfo.ResolvedEntryStackDepth < 0) { + continue; + } + if (blocksShareCompatibleDynamicJumpShapeClass(EntryPC, Info.EntryPC)) { + appendUniqueBlockPC(SourceBlockPCs, EntryPC); + } + } + return SourceBlockPCs; + } + + if (Info.DynamicJumpTargetRegions.empty()) { + if (!HasUnknownDynamicJump || !Info.IsDynamicJumpTargetCandidate) { + return SourceBlockPCs; + } + for (const auto &[EntryPC, RegionSourceInfo] : BlockInfos) { + if (!RegionSourceInfo.HasDynamicJump || + RegionSourceInfo.ResolvedEntryStackDepth < 0) { + continue; + } + appendUniqueBlockPC(SourceBlockPCs, EntryPC); + } + return SourceBlockPCs; + } + + for (uint64_t RegionEntryPC : Info.DynamicJumpTargetRegions) { + for (const auto &[EntryPC, RegionSourceInfo] : BlockInfos) { + if (!RegionSourceInfo.HasDynamicJump || + RegionSourceInfo.DynamicJumpTargetRegionEntryPC != RegionEntryPC) { + continue; + } + appendUniqueBlockPC(SourceBlockPCs, EntryPC); + } + } + return SourceBlockPCs; + } + + bool getUniformDynamicJumpEntryDepthForRegion(uint64_t RegionEntryPC, + int32_t &EntryDepth) const { + bool SawDynamicJump = false; + for (const auto &[EntryPC, Info] : BlockInfos) { + (void)EntryPC; + if (!Info.HasDynamicJump || + Info.DynamicJumpTargetRegionEntryPC != RegionEntryPC) { + continue; + } + if (Info.ResolvedEntryStackDepth < 0) { + continue; + } + if (Info.ResolvedExitStackDepth < 0) { + return false; + } + if (!SawDynamicJump) { + EntryDepth = Info.ResolvedExitStackDepth; + SawDynamicJump = true; + continue; + } + if (EntryDepth != Info.ResolvedExitStackDepth) { + return false; + } + } + return SawDynamicJump; + } + + void markDynamicJumpTargetCandidates() { + for (auto &[EntryPC, Info] : BlockInfos) { + (void)EntryPC; + Info.IsDynamicJumpTargetCandidate = false; + Info.HasCompatibleDynamicJumpTargetShape = false; + Info.DynamicJumpTargetRegions.clear(); + } + + if (!HasUnknownDynamicJump) { + return; + } + + const std::vector Regions = collectReachableDynamicJumpRegions(); + if (Regions.empty()) { + for (auto &[EntryPC, Info] : BlockInfos) { + (void)EntryPC; + if (Info.IsJumpDest) { + Info.IsDynamicJumpTargetCandidate = true; + } + } + return; + } + + for (uint64_t RegionEntryPC : Regions) { + std::queue WorkList; + std::map Visited; + Visited[RegionEntryPC] = true; + WorkList.push(RegionEntryPC); + + while (!WorkList.empty()) { + uint64_t BlockPC = WorkList.front(); + WorkList.pop(); + auto It = BlockInfos.find(BlockPC); + if (It == BlockInfos.end()) { + continue; + } + auto &Info = It->second; + if (Info.IsJumpDest) { + Info.IsDynamicJumpTargetCandidate = true; + addDynamicJumpRegion(Info, RegionEntryPC); + } + for (uint64_t SuccPC : Info.Successors) { + if (Visited.emplace(SuccPC, true).second) { + WorkList.push(SuccPC); } } } } - // Finalize last block and consecutive run - JITResult.MaxConsecutiveExpensive = - std::max(JITResult.MaxConsecutiveExpensive, CurConsecutiveExpensive); - if (BlockInfos.count(CurInfo.EntryPC) == 0) { - JITResult.MaxBlockExpensiveCount = - std::max(JITResult.MaxBlockExpensiveCount, - static_cast(CurInfo.RAExpensiveCount)); - BlockInfos.emplace(CurInfo.EntryPC, CurInfo); + + // The runtime indirect-jump lowering validates against the full + // JUMPDEST table, not just blocks reachable from the analyzer's + // fallthrough region approximation. Any remaining JUMPDEST must therefore + // stay on the conservative dynamic-target path. + for (auto &[EntryPC, Info] : BlockInfos) { + (void)EntryPC; + if (Info.IsJumpDest && !Info.IsDynamicJumpTargetCandidate) { + Info.IsDynamicJumpTargetCandidate = true; + } } + } - // Compute final fallback verdict - JITResult.ShouldFallback = - BytecodeSize > MAX_JIT_BYTECODE_SIZE || - JITResult.MirEstimate > MAX_JIT_MIR_ESTIMATE || - JITResult.MaxConsecutiveExpensive > MAX_CONSECUTIVE_RA_EXPENSIVE || - JITResult.MaxBlockExpensiveCount > MAX_BLOCK_RA_EXPENSIVE || - JITResult.DupFeedbackPatternCount > MAX_DUP_FEEDBACK_PATTERN; + struct DynamicJumpTargetShape { + int32_t FullEntryStateDepth = -1; + int32_t HiddenLiveInPrefixDepth = 0; + bool RequiresEntryMergeState = false; - return true; + bool operator==(const DynamicJumpTargetShape &Other) const { + return FullEntryStateDepth == Other.FullEntryStateDepth && + HiddenLiveInPrefixDepth == Other.HiddenLiveInPrefixDepth && + RequiresEntryMergeState == Other.RequiresEntryMergeState; + } + }; + + struct DynamicJumpShapeClassKey { + int32_t FullEntryStateDepth = -1; + int32_t HiddenLiveInPrefixDepth = 0; + bool RequiresEntryMergeState = false; + + bool operator<(const DynamicJumpShapeClassKey &Other) const { + if (FullEntryStateDepth != Other.FullEntryStateDepth) { + return FullEntryStateDepth < Other.FullEntryStateDepth; + } + if (HiddenLiveInPrefixDepth != Other.HiddenLiveInPrefixDepth) { + return HiddenLiveInPrefixDepth < Other.HiddenLiveInPrefixDepth; + } + return RequiresEntryMergeState < Other.RequiresEntryMergeState; + } + }; + + void resolveDynamicJumpTargetEntryDepths() { + if (!HasUnknownDynamicJump) { + return; + } + + for (uint64_t RegionEntryPC : collectReachableDynamicJumpRegions()) { + int32_t DynamicJumpEntryDepth = -1; + if (!getUniformDynamicJumpEntryDepthForRegion(RegionEntryPC, + DynamicJumpEntryDepth)) { + continue; + } + + std::queue WorkList; + for (auto &[EntryPC, Info] : BlockInfos) { + (void)EntryPC; + if (!hasDynamicJumpRegion(Info, RegionEntryPC) || + Info.HasInconsistentEntryDepth) { + continue; + } + if (Info.ResolvedEntryStackDepth < 0) { + Info.ResolvedEntryStackDepth = DynamicJumpEntryDepth; + WorkList.push(Info.EntryPC); + continue; + } + if (Info.ResolvedEntryStackDepth != DynamicJumpEntryDepth) { + invalidateReachableEntryDepths(Info.EntryPC); + } + } + + propagateEntryDepths(WorkList); + } + } + + bool hasCompatibleDynamicJumpTargetsForRegion(uint64_t RegionEntryPC) const { + int32_t DynamicJumpEntryDepth = -1; + if (!getUniformDynamicJumpEntryDepthForRegion(RegionEntryPC, + DynamicJumpEntryDepth)) { + return false; + } + DynamicJumpTargetShape ExpectedShape; + bool SawJumpDest = false; + for (const auto &[EntryPC, Info] : BlockInfos) { + (void)EntryPC; + if (!hasDynamicJumpRegion(Info, RegionEntryPC)) { + continue; + } + if (!Info.IsEntryStateCompatible) { + return false; + } + if (Info.FullEntryStateDepth != DynamicJumpEntryDepth) { + return false; + } + + DynamicJumpTargetShape CurrentShape = { + Info.FullEntryStateDepth, + Info.HiddenLiveInPrefixDepth, + Info.RequiresEntryMergeState, + }; + if (!SawJumpDest) { + ExpectedShape = CurrentShape; + SawJumpDest = true; + continue; + } + if (!(ExpectedShape == CurrentShape)) { + return false; + } + } + return SawJumpDest; + } + + bool hasGloballyIncompatibleDynamicJumpSource(uint64_t TargetBlockPC) const { + auto It = BlockInfos.find(TargetBlockPC); + if (It == BlockInfos.end() || !It->second.IsDynamicJumpTargetCandidate || + !It->second.HasCompatibleDynamicJumpTargetShape) { + return false; + } + + const std::vector TargetRegions = + It->second.DynamicJumpTargetRegions; + for (const auto &[EntryPC, Info] : BlockInfos) { + if (!Info.HasDynamicJump || Info.ResolvedEntryStackDepth < 0 || + Info.DynamicJumpTargetRegionEntryPC == 0) { + continue; + } + if (!TargetRegions.empty() && + std::find(TargetRegions.begin(), TargetRegions.end(), + Info.DynamicJumpTargetRegionEntryPC) == + TargetRegions.end()) { + continue; + } + if (!blocksShareCompatibleDynamicJumpShapeClass(EntryPC, TargetBlockPC)) { + return true; + } + } + return false; + } + + void finalizeLiftability() { + for (auto &[EntryPC, Info] : BlockInfos) { + (void)EntryPC; + bool EntryKnown = Info.IsEntryStateCompatible; + bool DynamicJumpDestConflict = HasUnknownDynamicJump && + Info.IsDynamicJumpTargetCandidate && + !Info.HasCompatibleDynamicJumpTargetShape; + Info.CanLiftStack = EntryKnown && !Info.HasUndefinedInstr && + !Info.HasInconsistentEntryDepth && + !DynamicJumpDestConflict; + if (Info.CanLiftStack && Info.IsDynamicJumpTargetCandidate && + Info.HasDeferredEntryMerge && Info.HiddenLiveInPrefixDepth > 0 && + getDynamicJumpSourceBlocksForBlock(EntryPC).empty()) { + Info.CanLiftStack = false; + } + } + } + + void finalizeEntryShapeMetadata() { + for (auto &[EntryPC, Info] : BlockInfos) { + (void)EntryPC; + Info.FullEntryStateDepth = Info.ResolvedEntryStackDepth; + Info.IsEntryStateCompatible = + Info.ResolvedEntryStackDepth >= 0 && !Info.HasInconsistentEntryDepth; + Info.HiddenLiveInPrefixDepth = 0; + Info.HasHiddenLiveInPrefix = false; + if (Info.IsEntryStateCompatible && + Info.ResolvedEntryStackDepth > Info.EntryStackDepth) { + Info.HiddenLiveInPrefixDepth = + Info.ResolvedEntryStackDepth - Info.EntryStackDepth; + Info.HasHiddenLiveInPrefix = Info.HiddenLiveInPrefixDepth > 0; + } + Info.RequiresEntryMergeState = + Info.IsEntryStateCompatible && + getPotentialEntryPredecessorsForBlock(EntryPC).size() > 1; + Info.HasDeferredEntryMerge = false; + } + + std::map CompatibleDynamicJumpRegions; + for (uint64_t RegionEntryPC : collectReachableDynamicJumpRegions()) { + CompatibleDynamicJumpRegions[RegionEntryPC] = + hasCompatibleDynamicJumpTargetsForRegion(RegionEntryPC); + } + + finalizeDynamicJumpRegionMetadata(CompatibleDynamicJumpRegions); + + for (auto &[EntryPC, Info] : BlockInfos) { + (void)EntryPC; + bool AllCompatibleRegions = Info.IsDynamicJumpTargetCandidate; + for (uint64_t RegionEntryPC : Info.DynamicJumpTargetRegions) { + auto It = CompatibleDynamicJumpRegions.find(RegionEntryPC); + if (It == CompatibleDynamicJumpRegions.end() || !It->second) { + AllCompatibleRegions = false; + break; + } + } + Info.HasCompatibleDynamicJumpTargetShape = AllCompatibleRegions; + if (Info.HasCompatibleDynamicJumpTargetShape && + hasGloballyIncompatibleDynamicJumpSource(EntryPC)) { + Info.HasCompatibleDynamicJumpTargetShape = false; + } + Info.HasDeferredEntryMerge = Info.HasCompatibleDynamicJumpTargetShape; + } + } + + void finalizeDynamicJumpRegionMetadata( + const std::map &CompatibleDynamicJumpRegions) { + DynamicJumpRegions.clear(); + + std::map ShapeClassIds; + uint32_t NextShapeClassId = 1; + + for (uint64_t RegionEntryPC : collectReachableDynamicJumpRegions()) { + auto &RegionInfo = DynamicJumpRegions[RegionEntryPC]; + RegionInfo.RegionEntryPC = RegionEntryPC; + RegionInfo.HasCompatibleTargetShape = + CompatibleDynamicJumpRegions.count(RegionEntryPC) != 0 && + CompatibleDynamicJumpRegions.at(RegionEntryPC); + RegionInfo.HasUniformEntryDepth = + getUniformDynamicJumpEntryDepthForRegion( + RegionEntryPC, RegionInfo.UniformEntryDepth); + + for (const auto &[EntryPC, Info] : BlockInfos) { + if (Info.HasDynamicJump && + Info.DynamicJumpTargetRegionEntryPC == RegionEntryPC) { + RegionInfo.SourceBlocks.push_back(EntryPC); + } + if (!hasDynamicJumpRegion(Info, RegionEntryPC)) { + continue; + } + RegionInfo.TargetBlocks.push_back(EntryPC); + if (!RegionInfo.HasCompatibleTargetShape) { + continue; + } + RegionInfo.FullEntryStateDepth = Info.FullEntryStateDepth; + RegionInfo.HiddenLiveInPrefixDepth = Info.HiddenLiveInPrefixDepth; + RegionInfo.RequiresEntryMergeState = Info.RequiresEntryMergeState; + } + + if (!RegionInfo.HasCompatibleTargetShape) { + continue; + } + + DynamicJumpShapeClassKey ShapeKey = { + RegionInfo.FullEntryStateDepth, + RegionInfo.HiddenLiveInPrefixDepth, + RegionInfo.RequiresEntryMergeState, + }; + auto [It, Inserted] = ShapeClassIds.emplace(ShapeKey, NextShapeClassId); + if (Inserted) { + ++NextShapeClassId; + } + RegionInfo.ShapeClassId = It->second; + } } -private: std::map BlockInfos; - uint64_t PC = 0; + std::map DynamicJumpRegions; + std::map JumpDestCanonicalPCs; + uint64_t EntryBlockPC = 0; + bool HasUnknownDynamicJump = false; evmc_revision Revision = zen::evm::DEFAULT_REVISION; JITSuitabilityResult JITResult; }; diff --git a/src/compiler/evm_frontend/evm_imported.cpp b/src/compiler/evm_frontend/evm_imported.cpp index ff5462c99..d33e30891 100644 --- a/src/compiler/evm_frontend/evm_imported.cpp +++ b/src/compiler/evm_frontend/evm_imported.cpp @@ -102,6 +102,42 @@ inline void triggerStaticModeViolation(zen::runtime::EVMInstance *Instance) { zen::runtime::EVMInstance::triggerInstanceExceptionOnJIT( Instance, zen::common::ErrorCode::EVMStaticModeViolation); } + +constexpr uint8_t DelegationMagicBytes[] = {0xef, 0x01, 0x00}; + +bool resolveDelegatedCallCodeAddress(zen::runtime::EVMInstance *Instance, + evmc::address TargetAddr, + evmc::address &CodeAddr) { + CodeAddr = TargetAddr; + if (Instance->getRevision() < EVMC_PRAGUE) { + return true; + } + + const zen::runtime::EVMModule *Module = Instance->getModule(); + ZEN_ASSERT(Module && Module->Host); + uint8_t Designation[sizeof(DelegationMagicBytes) + sizeof(evmc::address)] = + {}; + const size_t Copied = + Module->Host->copy_code(TargetAddr, 0, Designation, sizeof(Designation)); + if (Copied < sizeof(DelegationMagicBytes) || + std::memcmp(Designation, DelegationMagicBytes, + sizeof(DelegationMagicBytes)) != 0) { + return true; + } + + if (Copied != sizeof(Designation)) { + return true; + } + + std::memcpy(CodeAddr.bytes, Designation + sizeof(DelegationMagicBytes), + sizeof(CodeAddr.bytes)); + const uint64_t DelegateAccessCost = + Module->Host->access_account(CodeAddr) == EVMC_ACCESS_COLD + ? zen::evm::COLD_ACCOUNT_ACCESS_COST + : zen::evm::WARM_STORAGE_READ_COST; + Instance->chargeGas(DelegateAccessCost); + return true; +} } // namespace const RuntimeFunctions &getRuntimeFunctionTable() { @@ -880,6 +916,12 @@ static uint64_t evmHandleCallInternal( Instance->chargeGas(zen::evm::ADDITIONAL_COLD_ACCOUNT_ACCESS_COST); } + evmc::address CodeAddr = TargetAddr; + if (!resolveDelegatedCallCodeAddress(Instance, TargetAddr, CodeAddr)) { + Instance->setReturnData({}); + return 0; + } + const bool HasValueArgs = CallKind == EVMC_CALL || CallKind == EVMC_CALLCODE; const bool HasValue = Value != 0; @@ -976,10 +1018,16 @@ static uint64_t evmHandleCallInternal( ? CurrentMsg->value : intx::be::store(Value), .create2_salt = {}, - .code_address = TargetAddr, + .code_address = CodeAddr, .code = nullptr, .code_size = 0, }; + if (std::memcmp(TargetAddr.bytes, CodeAddr.bytes, sizeof(TargetAddr.bytes)) != + 0) { + CallMsg.flags |= EVMC_DELEGATED; + } else { + CallMsg.flags &= ~uint32_t(EVMC_DELEGATED); + } Instance->pushMessage(&CallMsg); evmc::Result Result = Module->Host->call(CallMsg); diff --git a/src/compiler/evm_frontend/evm_lifted_stack_lifter.h b/src/compiler/evm_frontend/evm_lifted_stack_lifter.h new file mode 100644 index 000000000..bbecb9348 --- /dev/null +++ b/src/compiler/evm_frontend/evm_lifted_stack_lifter.h @@ -0,0 +1,480 @@ +// Copyright (C) 2025 the DTVM authors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#ifndef EVM_FRONTEND_EVM_LIFTED_STACK_LIFTER_H +#define EVM_FRONTEND_EVM_LIFTED_STACK_LIFTER_H + +#include "compiler/evm_frontend/evm_analyzer.h" + +#include +#include +#include +#include +#include + +namespace COMPILER { + +template class EVMLiftedStackLifter { +public: + using Operand = typename IRBuilder::Operand; + + struct StackValueId { + uint64_t Value = 0; + + bool operator==(const StackValueId &Other) const { + return Value == Other.Value; + } + + bool operator!=(const StackValueId &Other) const { + return !(*this == Other); + } + }; + + struct StackValue { + StackValueId Id; + Operand Value; + }; + + struct VirtualStackState { + std::vector Slots; + }; + + struct EdgeState { + uint64_t PredBlockPC = 0; + VirtualStackState StackState; + }; + + struct PendingPhi { + enum class ResolutionKind : uint8_t { + Pending, + Folded, + RequiresMaterialization, + }; + + uint32_t SlotIndex = 0; + std::map IncomingValues; + std::map IncomingPhiValues; + ResolutionKind Resolution = ResolutionKind::Pending; + bool IsComplete = false; + StackValueId FoldedValueId = {}; + }; + + struct MergeIncomingValue { + uint64_t PredBlockPC; + StackValueId ValueId; + Operand Value; + }; + + struct MergeMaterializationRequest { + uint64_t BlockPC; + uint32_t SlotIndex; + Operand MergeOperand; + std::vector ExpectedPredBlockPCs; + std::vector IncomingValues; + typename PendingPhi::ResolutionKind Resolution = + PendingPhi::ResolutionKind::Pending; + bool IsComplete = false; + }; + + struct BlockEntryState { + size_t EntryDepth = 0; + std::vector EntryOperands; + std::vector MergeOperands; + std::map IncomingStates; + VirtualStackState ResolvedEntryState; + std::vector PendingPhis; + std::vector PredecessorOrder; + size_t ExpectedIncomingCount = 0; + }; + + explicit EVMLiftedStackLifter(IRBuilder &Builder) : Builder(Builder) {} + + void initialize(const EVMAnalyzer &Analyzer) { + LiftedBlocks.clear(); + BlockEntryStates.clear(); + ValueIds.clear(); + NextValueId = 1; + NextOpaqueValueId = 1; +#ifdef ZEN_ENABLE_EVM_STACK_SSA_LIFT + for (const auto &[EntryPC, BlockInfo] : Analyzer.getBlockInfos()) { + if (!BlockInfo.CanLiftStack) { + continue; + } + LiftedBlocks[EntryPC] = true; + auto &EntryState = BlockEntryStates[EntryPC]; + EntryState.PredecessorOrder = + Analyzer.getPotentialEntryPredecessorsForBlock(EntryPC); + EntryState.ExpectedIncomingCount = EntryState.PredecessorOrder.size(); + EntryState.EntryDepth = + static_cast(std::max(BlockInfo.FullEntryStateDepth, 0)); + if (BlockInfo.FullEntryStateDepth <= 0) { + continue; + } + EntryState.EntryOperands.reserve( + static_cast(BlockInfo.FullEntryStateDepth)); + for (int32_t Depth = 0; Depth < BlockInfo.FullEntryStateDepth; ++Depth) { + EntryState.EntryOperands.push_back(Builder.createStackEntryOperand()); + } + EntryState.ResolvedEntryState = + makeVirtualStackState(EntryState.EntryOperands); + if (BlockInfo.RequiresEntryMergeState) { + EntryState.MergeOperands.resize(EntryState.EntryDepth); + EntryState.PendingPhis.reserve(EntryState.EntryDepth); + for (size_t SlotIndex = 0; SlotIndex < EntryState.EntryDepth; + ++SlotIndex) { + EntryState.PendingPhis.push_back( + PendingPhi{static_cast(SlotIndex), {}, {}}); + } + } + } +#else + (void)Analyzer; +#endif + } + + bool isLiftedBlock(uint64_t BlockPC) const { +#ifdef ZEN_ENABLE_EVM_STACK_SSA_LIFT + auto It = LiftedBlocks.find(BlockPC); + return It != LiftedBlocks.end() && It->second; +#else + (void)BlockPC; + return false; +#endif + } + + const std::vector *getEntryState(uint64_t BlockPC) const { + const auto *EntryState = getBlockEntryState(BlockPC); + if (!EntryState) { + return nullptr; + } + return &EntryState->EntryOperands; + } + + bool hasCompleteEntryState(uint64_t BlockPC) const { +#ifdef ZEN_ENABLE_EVM_STACK_SSA_LIFT + const auto *EntryState = getBlockEntryState(BlockPC); + if (!EntryState) { + return false; + } + if (EntryState->ExpectedIncomingCount == 0) { + return true; + } + if (EntryState->IncomingStates.size() < EntryState->ExpectedIncomingCount) { + return false; + } + for (const PendingPhi &Phi : EntryState->PendingPhis) { + if (!Phi.IsComplete) { + return false; + } + } + return true; +#else + (void)BlockPC; + return false; +#endif + } + + std::vector getLogicalEntryState(uint64_t BlockPC) const { +#ifdef ZEN_ENABLE_EVM_STACK_SSA_LIFT + std::vector LogicalState; + const auto *EntryState = getBlockEntryState(BlockPC); + if (!EntryState) { + return LogicalState; + } + + LogicalState = EntryState->EntryOperands; + if (LogicalState.empty() && !EntryState->ResolvedEntryState.Slots.empty()) { + LogicalState.reserve(EntryState->ResolvedEntryState.Slots.size()); + for (const StackValue &Slot : EntryState->ResolvedEntryState.Slots) { + LogicalState.push_back(Slot.Value); + } + } + for (const PendingPhi &Phi : EntryState->PendingPhis) { + ZEN_ASSERT(Phi.SlotIndex < LogicalState.size()); + if (Phi.SlotIndex < EntryState->MergeOperands.size() && + !EntryState->MergeOperands[Phi.SlotIndex].isEmpty()) { + LogicalState[Phi.SlotIndex] = EntryState->MergeOperands[Phi.SlotIndex]; + continue; + } + if (Phi.Resolution == PendingPhi::ResolutionKind::Folded && + Phi.IsComplete && !Phi.IncomingValues.empty()) { + auto It = Phi.IncomingValues.begin(); + ZEN_ASSERT(It != Phi.IncomingValues.end()); + LogicalState[Phi.SlotIndex] = It->second.Value; + continue; + } + } + return LogicalState; +#else + (void)BlockPC; + return {}; +#endif + } + + std::vector + getMergeMaterializationRequests(uint64_t BlockPC) const { +#ifdef ZEN_ENABLE_EVM_STACK_SSA_LIFT + std::vector Requests; + const auto *EntryState = getBlockEntryState(BlockPC); + if (!EntryState) { + return Requests; + } + + Requests.reserve(EntryState->PendingPhis.size()); + for (const PendingPhi &Phi : EntryState->PendingPhis) { + if (Phi.IsComplete && + Phi.Resolution == PendingPhi::ResolutionKind::Folded) { + continue; + } + + ZEN_ASSERT(Phi.SlotIndex < EntryState->MergeOperands.size()); + MergeMaterializationRequest Request = { + BlockPC, + Phi.SlotIndex, + EntryState->MergeOperands[Phi.SlotIndex], + EntryState->PredecessorOrder, + {}, + Phi.Resolution, + Phi.IsComplete, + }; + Request.IncomingValues.reserve(Phi.IncomingPhiValues.size()); + for (const auto &[PredBlockPC, IncomingValue] : Phi.IncomingValues) { + auto PhiValueIt = Phi.IncomingPhiValues.find(PredBlockPC); + ZEN_ASSERT(PhiValueIt != Phi.IncomingPhiValues.end()); + Request.IncomingValues.push_back( + {PredBlockPC, IncomingValue.Id, PhiValueIt->second}); + } + Requests.push_back(Request); + } + return Requests; +#else + (void)BlockPC; + return {}; +#endif + } + + const BlockEntryState *getBlockEntryState(uint64_t BlockPC) const { +#ifdef ZEN_ENABLE_EVM_STACK_SSA_LIFT + auto It = BlockEntryStates.find(BlockPC); + if (It == BlockEntryStates.end()) { + return nullptr; + } + return &It->second; +#else + (void)BlockPC; + return nullptr; +#endif + } + + void assignEntryState(uint64_t PredBlockPC, uint64_t BlockPC, + const std::vector &Values) { +#ifdef ZEN_ENABLE_EVM_STACK_SSA_LIFT + auto It = BlockEntryStates.find(BlockPC); + if (It == BlockEntryStates.end()) { + return; + } + auto &EntryState = It->second; + ZEN_ASSERT(EntryState.EntryDepth == Values.size() && + "Lifted entry state size mismatch"); + VirtualStackState State = makeVirtualStackState(Values); + EntryState.IncomingStates[PredBlockPC] = EdgeState{PredBlockPC, State}; + EntryState.ResolvedEntryState = State; + for (size_t Index = 0; Index < EntryState.EntryDepth; ++Index) { + if (Index < EntryState.PendingPhis.size()) { + EntryState.PendingPhis[Index].IncomingValues[PredBlockPC] = + State.Slots[Index]; + EntryState.PendingPhis[Index].IncomingPhiValues[PredBlockPC] = + prepareStackPhiIncomingCompat(Values[Index]); + updatePendingPhi(EntryState.PendingPhis[Index], + EntryState.ExpectedIncomingCount); + } + if (Index < EntryState.MergeOperands.size()) { + if (!EntryState.MergeOperands[Index].isEmpty()) { + assignStackMergeOperandCompat( + EntryState.MergeOperands[Index], PredBlockPC, + EntryState.PendingPhis[Index].IncomingPhiValues[PredBlockPC]); + } + } else if (Index < EntryState.EntryOperands.size()) { + Builder.assignStackEntryOperand(EntryState.EntryOperands[Index], + Values[Index]); + } + } +#else + (void)PredBlockPC; + (void)BlockPC; + (void)Values; +#endif + } + + void assignMergeOperand(uint64_t BlockPC, uint32_t SlotIndex, + const Operand &MergeOperand) { +#ifdef ZEN_ENABLE_EVM_STACK_SSA_LIFT + auto It = BlockEntryStates.find(BlockPC); + if (It == BlockEntryStates.end()) { + return; + } + auto &EntryState = It->second; + ZEN_ASSERT(SlotIndex < EntryState.MergeOperands.size()); + EntryState.MergeOperands[SlotIndex] = MergeOperand; +#else + (void)BlockPC; + (void)SlotIndex; + (void)MergeOperand; +#endif + } + +private: + template + struct HasGetInstr : std::false_type {}; + template + struct HasGetInstr< + T, std::void_t().getInstr())>> + : std::true_type {}; + + template struct HasGetVar : std::false_type {}; + template + struct HasGetVar().getVar())>> + : std::true_type {}; + + template + struct HasGetU256VarComponents : std::false_type {}; + template + struct HasGetU256VarComponents< + T, + std::void_t().getU256VarComponents())>> + : std::true_type {}; + + template + struct HasPrepareStackPhiIncoming : std::false_type {}; + template + struct HasPrepareStackPhiIncoming< + T, std::void_t().prepareStackPhiIncoming( + std::declval()))>> : std::true_type {}; + + template + struct HasAssignStackMergeOperand : std::false_type {}; + template + struct HasAssignStackMergeOperand< + T, std::void_t().assignStackMergeOperand( + std::declval(), uint64_t{}, + std::declval()))>> : std::true_type {}; + + Operand prepareStackPhiIncomingCompat(const Operand &Value) { + if constexpr (HasPrepareStackPhiIncoming::value) { + return Builder.prepareStackPhiIncoming(Value); + } else { + return Value; + } + } + + void assignStackMergeOperandCompat(const Operand &Dest, uint64_t PredBlockPC, + const Operand &Value) { + if constexpr (HasAssignStackMergeOperand::value) { + Builder.assignStackMergeOperand(Dest, PredBlockPC, Value); + } else { + (void)PredBlockPC; + Builder.assignStackEntryOperand(Dest, Value); + } + } + + static void appendPointerKey(std::string &Key, const void *Ptr) { + Key += std::to_string(reinterpret_cast(Ptr)); + } + + std::string getOperandIdentityKey(const Operand &Value) { + if (Value.isConstant()) { + const auto &ConstValue = Value.getConstValue(); + std::string Key = "const"; + for (uint64_t Word : ConstValue) { + Key += ':'; + Key += std::to_string(Word); + } + return Key; + } + + if constexpr (HasGetInstr::value) { + if (auto *Instr = Value.getInstr()) { + std::string Key = "instr:"; + appendPointerKey(Key, Instr); + return Key; + } + } + + if constexpr (HasGetVar::value) { + if (auto *Var = Value.getVar()) { + std::string Key = "var:"; + appendPointerKey(Key, Var); + return Key; + } + } + + if constexpr (HasGetU256VarComponents::value) { + const auto &VarComponents = Value.getU256VarComponents(); + std::string Key = "u256vars"; + for (const auto *Var : VarComponents) { + Key += ':'; + appendPointerKey(Key, Var); + } + return Key; + } + + std::string Key = "opaque:"; + Key += std::to_string(NextOpaqueValueId++); + return Key; + } + + StackValueId getOrCreateStackValueId(const Operand &Value) { + const std::string Key = getOperandIdentityKey(Value); + auto It = ValueIds.find(Key); + if (It != ValueIds.end()) { + return StackValueId{It->second}; + } + const uint64_t NewId = NextValueId++; + ValueIds.emplace(Key, NewId); + return StackValueId{NewId}; + } + + void updatePendingPhi(PendingPhi &Phi, size_t ExpectedIncomingCount) { + Phi.IsComplete = ExpectedIncomingCount > 0 && + Phi.IncomingValues.size() >= ExpectedIncomingCount; + Phi.FoldedValueId = {}; + if (!Phi.IsComplete) { + Phi.Resolution = PendingPhi::ResolutionKind::Pending; + return; + } + + auto It = Phi.IncomingValues.begin(); + ZEN_ASSERT(It != Phi.IncomingValues.end()); + StackValueId CandidateId = It->second.Id; + ++It; + for (; It != Phi.IncomingValues.end(); ++It) { + if (It->second.Id != CandidateId) { + Phi.Resolution = PendingPhi::ResolutionKind::RequiresMaterialization; + return; + } + } + + Phi.Resolution = PendingPhi::ResolutionKind::Folded; + Phi.FoldedValueId = CandidateId; + } + + VirtualStackState makeVirtualStackState(const std::vector &Values) { + VirtualStackState State; + State.Slots.reserve(Values.size()); + for (const Operand &Value : Values) { + State.Slots.push_back(StackValue{getOrCreateStackValueId(Value), Value}); + } + return State; + } + + IRBuilder &Builder; + std::map LiftedBlocks; + std::map BlockEntryStates; + std::map ValueIds; + uint64_t NextValueId = 1; + uint64_t NextOpaqueValueId = 1; +}; + +} // namespace COMPILER + +#endif // EVM_FRONTEND_EVM_LIFTED_STACK_LIFTER_H diff --git a/src/compiler/evm_frontend/evm_mir_compiler.cpp b/src/compiler/evm_frontend/evm_mir_compiler.cpp index f1a771bfe..a60fb8555 100644 --- a/src/compiler/evm_frontend/evm_mir_compiler.cpp +++ b/src/compiler/evm_frontend/evm_mir_compiler.cpp @@ -8,7 +8,10 @@ #include "evm/gas_storage_cost.h" #include "runtime/evm_instance.h" #include "utils/hash_utils.h" +#include #include +#include +#include #ifdef ZEN_ENABLE_EVM_GAS_REGISTER #include "compiler/llvm-prebuild/Target/X86/X86Subtarget.h" @@ -89,6 +92,103 @@ bool EVMMirBuilder::compile(CompilerContext *Context) { return Visitor.compile(); } +void EVMMirBuilder::registerDynamicJumpPhiIncomingBlock(uint64_t TargetBlockPC, + uint64_t PredBlockPC, + MBasicBlock *PredBB) { + registerPhiIncomingBlock(TargetBlockPC, PredBlockPC, PredBB); +} + +void EVMMirBuilder::registerPhiIncomingBlock(uint64_t TargetBlockPC, + uint64_t PredBlockPC, + MBasicBlock *PredBB) { + const uint64_t CanonicalTargetPC = getCanonicalJumpDestPC(TargetBlockPC); + DynamicPhiIncomingBlockTable[CanonicalTargetPC][PredBlockPC] = + resolvePhiIncomingPredecessorBB(TargetBlockPC, PredBB); +} + +MBasicBlock *EVMMirBuilder::getPhiIncomingBlock(uint64_t TargetBlockPC, + uint64_t PredBlockPC) const { + auto DynamicTargetIt = + DynamicPhiIncomingBlockTable.find(getCanonicalJumpDestPC(TargetBlockPC)); + if (DynamicTargetIt != DynamicPhiIncomingBlockTable.end()) { + auto DynamicPredIt = DynamicTargetIt->second.find(PredBlockPC); + if (DynamicPredIt != DynamicTargetIt->second.end()) { + return resolveReachablePhiIncomingPredecessorBB(TargetBlockPC, + DynamicPredIt->second); + } + } + + auto BlockIt = BlockEntryTable.find(PredBlockPC); + if (BlockIt == BlockEntryTable.end()) { + return nullptr; + } + return resolveReachablePhiIncomingPredecessorBB(TargetBlockPC, + BlockIt->second); +} + +uint64_t EVMMirBuilder::getCanonicalJumpDestPC(uint64_t TargetBlockPC) const { + auto It = JumpDestCanonicalPCTable.find(TargetBlockPC); + return It == JumpDestCanonicalPCTable.end() ? TargetBlockPC : It->second; +} + +MBasicBlock *EVMMirBuilder::resolvePhiIncomingPredecessorBB( + uint64_t TargetBlockPC, MBasicBlock *DirectPredBB) const { + const uint64_t CanonicalTargetPC = getCanonicalJumpDestPC(TargetBlockPC); + auto BodyIt = JumpDestBodyTable.find(CanonicalTargetPC); + if (BodyIt == JumpDestBodyTable.end()) { + return DirectPredBB; + } + + auto EntryIt = JumpDestTable.find(TargetBlockPC); + if (EntryIt == JumpDestTable.end()) { + return DirectPredBB; + } + + return EntryIt->second == BodyIt->second ? DirectPredBB : EntryIt->second; +} + +MBasicBlock *EVMMirBuilder::resolveReachablePhiIncomingPredecessorBB( + uint64_t TargetBlockPC, MBasicBlock *CandidateBB) const { + if (CandidateBB == nullptr) { + return nullptr; + } + + auto TargetIt = BlockEntryTable.find(TargetBlockPC); + if (TargetIt == BlockEntryTable.end()) { + return CandidateBB; + } + + MBasicBlock *TargetBB = TargetIt->second; + auto PredRange = TargetBB->predecessors(); + if (std::find(PredRange.begin(), PredRange.end(), CandidateBB) != + PredRange.end()) { + return CandidateBB; + } + + std::queue Worklist; + std::set Visited; + Worklist.push(CandidateBB); + Visited.insert(CandidateBB); + + while (!Worklist.empty()) { + MBasicBlock *CurrentBB = Worklist.front(); + Worklist.pop(); + + for (MBasicBlock *SuccBB : CurrentBB->successors()) { + if (SuccBB == nullptr || !Visited.insert(SuccBB).second) { + continue; + } + if (std::find(PredRange.begin(), PredRange.end(), SuccBB) != + PredRange.end()) { + return SuccBB; + } + Worklist.push(SuccBB); + } + } + + return CandidateBB; +} + void EVMMirBuilder::loadEVMInstanceAttr() { InstanceAddr = createInstruction( false, OP_ptrtoint, &Ctx.I64Type, @@ -135,13 +235,15 @@ void EVMMirBuilder::loadEVMInstanceAttr() { ExceptionReturnBB = CurFunc->createExceptionReturnBB(); } -MBasicBlock *EVMMirBuilder::getOrCreateIndirectJumpBB() { - if (IndirectJumpBB) { - return IndirectJumpBB; +MBasicBlock *EVMMirBuilder::getOrCreateIndirectJumpBB(uint64_t SourceBlockPC) { + auto ExistingIt = IndirectJumpBBs.find(SourceBlockPC); + if (ExistingIt != IndirectJumpBBs.end()) { + return ExistingIt->second; } MBasicBlock *FromBB = CurBB; - IndirectJumpBB = CurFunc->createBasicBlock(); + MBasicBlock *IndirectJumpBB = CurFunc->createBasicBlock(); + IndirectJumpBBs[SourceBlockPC] = IndirectJumpBB; setInsertBlock(IndirectJumpBB); #ifdef ZEN_ENABLE_LINUX_PERF CurBB->setSourceOffset(CurPC); @@ -200,6 +302,8 @@ MBasicBlock *EVMMirBuilder::getOrCreateIndirectJumpBB() { false, CmpInstruction::Predicate::ICMP_EQ, &Ctx.I64Type, JumpTarget, ExpectedPC); MBasicBlock *DestBB = JumpHashTable[HashEntry][0]; + registerDynamicJumpPhiIncomingBlock(JumpHashReverse[HashEntry][0], + SourceBlockPC, CheckBB); createInstruction(true, Ctx, IsMatch, DestBB, FailureBB); addSuccessor(DestBB); @@ -222,6 +326,8 @@ MBasicBlock *EVMMirBuilder::getOrCreateIndirectJumpBB() { SubCases[I].first = createIntConstInstruction(UInt64Type, SubPCVec[I]); SubCases[I].second = SubDestBBVec[I]; + registerDynamicJumpPhiIncomingBlock(SubPCVec[I], SourceBlockPC, + SubCaseBB); addSuccessor(SubDestBBVec[I]); } createInstruction(true, Ctx, JumpTarget, FailureBB, @@ -247,6 +353,7 @@ MBasicBlock *EVMMirBuilder::getOrCreateIndirectJumpBB() { for (const auto &[DestPC, DestBB] : JumpDestTable) { Cases[Index].first = createIntConstInstruction(UInt64Type, DestPC); Cases[Index].second = DestBB; + registerDynamicJumpPhiIncomingBlock(DestPC, SourceBlockPC, IndirectJumpBB); addSuccessor(DestBB); Index++; } @@ -876,6 +983,184 @@ typename EVMMirBuilder::Operand EVMMirBuilder::stackGet(int32_t IndexFromTop) { return Operand(GetComponents, EVMType::UINT256); } +void EVMMirBuilder::setTrackedStackDepth(uint32_t Depth) { + MType *I64Type = EVMFrontendContext::getMIRTypeFromEVMType(EVMType::UINT64); + uint64_t StackBytes = static_cast(Depth) * 32ULL; + MInstruction *StackSize = createIntConstInstruction(I64Type, StackBytes); + createInstruction(true, &(Ctx.VoidType), StackSize, + StackSizeVar->getVarIdx()); + + MInstruction *StackPtrOffset = createIntConstInstruction( + &Ctx.I64Type, zen::runtime::EVMInstance::getEVMStackOffset()); + MInstruction *StackBaseAddr = createInstruction( + false, OP_add, &Ctx.I64Type, InstanceAddr, StackPtrOffset); + MInstruction *StackTopAddr = createInstruction( + false, OP_add, &Ctx.I64Type, StackBaseAddr, StackSize); + createInstruction(true, &(Ctx.VoidType), StackTopAddr, + StackTopVar->getVarIdx()); +} + +typename EVMMirBuilder::Operand EVMMirBuilder::createStackEntryOperand() { + U256Var Vars = {}; + for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) { + Vars[I] = CurFunc->createVariable(&Ctx.I64Type); + } + return Operand(Vars, EVMType::UINT256); +} + +void EVMMirBuilder::assignStackEntryOperand(const Operand &Dest, + const Operand &Value) { + ZEN_ASSERT(Dest.isU256MultiComponent() && "stack entry operand must be U256"); + U256Var DestVars = Dest.getU256VarComponents(); + U256Inst Src = extractU256Operand(Value); + for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) { + ZEN_ASSERT(DestVars[I] != nullptr); + createInstruction(true, &(Ctx.VoidType), Src[I], + DestVars[I]->getVarIdx()); + } +} + +typename EVMMirBuilder::Operand +EVMMirBuilder::prepareStackPhiIncoming(const Operand &Value) { + U256Inst Prepared = {}; + U256Inst Src = extractU256Operand(Value); + for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) { + Prepared[I] = protectUnsafeValue(Src[I], &Ctx.I64Type); + } + return Operand(Prepared, EVMType::UINT256); +} + +void EVMMirBuilder::registerCurrentBlockPC(uint64_t BlockPC) { + CurrentBlockPC = BlockPC; + BlockEntryTable[BlockPC] = CurBB; +} + +typename EVMMirBuilder::Operand EVMMirBuilder::materializeStackMergeOperand( + const std::vector &PredBlockPCs, + const std::vector> &IncomingValues) { + std::map IncomingValueMap; + for (const auto &[PredBlockPC, Value] : IncomingValues) { + IncomingValueMap[PredBlockPC] = Value; + } + + U256Inst PhiComponents = {}; + U256Var PhiVars = {}; + auto PredRange = CurBB->predecessors(); + const size_t ActualPredCount = + static_cast(std::distance(PredRange.begin(), PredRange.end())); + for (size_t ComponentIndex = 0; ComponentIndex < EVM_ELEMENTS_COUNT; + ++ComponentIndex) { + PhiInstruction *Phi = createPendingPhi(&Ctx.I64Type, PredBlockPCs.size()); + auto &SlotMap = PhiIncomingSlotMap[Phi]; + for (size_t IncomingIndex = 0; IncomingIndex < PredBlockPCs.size(); + ++IncomingIndex) { + uint64_t PredBlockPC = PredBlockPCs[IncomingIndex]; + SlotMap[PredBlockPC] = IncomingIndex; + + auto IncomingIt = IncomingValueMap.find(PredBlockPC); + if (IncomingIt == IncomingValueMap.end()) { + continue; + } + + MBasicBlock *IncomingBB = + getPhiIncomingBlock(CurrentBlockPC, PredBlockPC); + if ((IncomingBB == nullptr || + std::find(PredRange.begin(), PredRange.end(), IncomingBB) == + PredRange.end()) && + IncomingIndex < ActualPredCount) { + IncomingBB = *(PredRange.begin() + IncomingIndex); + } + ZEN_ASSERT( + IncomingBB != nullptr && + "phi incoming block must be registered before materialization"); + U256Inst IncomingComponents = extractU256Operand(IncomingIt->second); + Phi->setIncoming(IncomingIndex, IncomingBB, + IncomingComponents[ComponentIndex]); + } + PhiComponents[ComponentIndex] = Phi; + } + + for (size_t ComponentIndex = 0; ComponentIndex < EVM_ELEMENTS_COUNT; + ++ComponentIndex) { + Variable *PhiVar = + storeInstructionInTemp(PhiComponents[ComponentIndex], &Ctx.I64Type); + PhiVars[ComponentIndex] = PhiVar; + StackMergePhiVarMap[PhiVar->getVarIdx()] = + llvm::cast(PhiComponents[ComponentIndex]); + } + + return Operand(PhiVars, EVMType::UINT256); +} + +void EVMMirBuilder::assignStackMergeOperand(const Operand &Dest, + uint64_t PredBlockPC, + const Operand &Value) { + U256Var DestVars = Dest.getU256VarComponents(); + U256Inst IncomingComponents = extractU256Operand(Value); + MBasicBlock *IncomingBB = getPhiIncomingBlock(CurrentBlockPC, PredBlockPC); + auto PredRange = CurBB->predecessors(); + const size_t ActualPredCount = + static_cast(std::distance(PredRange.begin(), PredRange.end())); + for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) { + ZEN_ASSERT(DestVars[I] != nullptr && + "stack merge operand must be anchored in temp vars"); + auto PhiIt = StackMergePhiVarMap.find(DestVars[I]->getVarIdx()); + ZEN_ASSERT(PhiIt != StackMergePhiVarMap.end() && + "phi temp var must resolve to pending phi"); + PhiInstruction *Phi = PhiIt->second; + size_t IncomingSlot = getPhiIncomingSlot(Phi, PredBlockPC); + if ((IncomingBB == nullptr || std::find(PredRange.begin(), PredRange.end(), + IncomingBB) == PredRange.end()) && + IncomingSlot < ActualPredCount) { + IncomingBB = *(PredRange.begin() + IncomingSlot); + } + ZEN_ASSERT(IncomingBB != nullptr && + "phi incoming block must be registered before patching"); + Phi->setIncoming(IncomingSlot, IncomingBB, IncomingComponents[I]); + } +} + +void EVMMirBuilder::spillTrackedStack( + const std::vector &TrackedStack) { + spillTrackedStackPreservingPrefix(TrackedStack, 0); +} + +void EVMMirBuilder::spillTrackedStackPreservingPrefix( + const std::vector &TrackedStack, uint32_t PrefixDepth) { + MType *I64Type = EVMFrontendContext::getMIRTypeFromEVMType(EVMType::UINT64); + MPointerType *U64PtrType = MPointerType::create(Ctx, Ctx.I64Type); + MInstruction *StackPtrOffset = createIntConstInstruction( + &Ctx.I64Type, zen::runtime::EVMInstance::getEVMStackOffset()); + MInstruction *StackBaseAddr = createInstruction( + false, OP_add, &Ctx.I64Type, InstanceAddr, StackPtrOffset); + + const int32_t InnerOffsets[EVM_ELEMENTS_COUNT] = {0, 8, 16, 24}; + const uint64_t PrefixBytes = static_cast(PrefixDepth) * 32ULL; + for (size_t Slot = 0; Slot < TrackedStack.size(); ++Slot) { + U256Inst Components = extractU256Operand(TrackedStack[Slot]); + uint64_t SlotOffset = PrefixBytes + static_cast(Slot) * 32ULL; + MInstruction *SlotOffsetInst = + createIntConstInstruction(I64Type, SlotOffset); + MInstruction *SlotAddr = createInstruction( + false, OP_add, &Ctx.I64Type, StackBaseAddr, SlotOffsetInst); + MInstruction *SlotPtr = createInstruction( + false, OP_inttoptr, U64PtrType, SlotAddr); + for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) { + createInstruction(true, &Ctx.VoidType, Components[I], + SlotPtr, InnerOffsets[I]); + } + } + + const uint32_t FinalDepth = + PrefixDepth + static_cast(TrackedStack.size()); + setTrackedStackDepth(FinalDepth); + const int32_t StackSizeOffset = + zen::runtime::EVMInstance::getEVMStackSizeOffset(); + MInstruction *StackSize = createIntConstInstruction( + I64Type, static_cast(FinalDepth) * 32ULL); + setInstanceElement(&Ctx.I64Type, StackSize, StackSizeOffset); +} + void EVMMirBuilder::handleStop() { auto Zero = createU256ConstOperand(intx::uint256{0}); handleReturn(Zero, Zero); @@ -960,6 +1245,7 @@ void EVMMirBuilder::createJumpTable() { size_t BytecodeSize = EvmCtx->getBytecodeSize(); JumpDestTable.clear(); + JumpDestCanonicalPCTable.clear(); JumpDestBodyTable.clear(); JumpHashTable.clear(); JumpHashReverse.clear(); @@ -988,6 +1274,7 @@ void EVMMirBuilder::createJumpTable() { BodyBB->setJumpDestBB(true); for (size_t DestPC = RangeStart; DestPC <= RangeEnd; ++DestPC) { + JumpDestCanonicalPCTable[DestPC] = static_cast(RangeEnd); JumpDestBodyTable[DestPC] = BodyBB; } @@ -1079,6 +1366,7 @@ void EVMMirBuilder::createJumpTable() { void EVMMirBuilder::implementConstantJump(uint64_t ConstDest, MBasicBlock *FailureBB) { if (JumpDestTable.count(ConstDest)) { + registerPhiIncomingBlock(ConstDest, CurrentBlockPC, CurBB); createInstruction(true, Ctx, JumpDestTable[ConstDest]); addSuccessor(JumpDestTable[ConstDest]); } else { @@ -1096,7 +1384,7 @@ void EVMMirBuilder::implementIndirectJump(MInstruction *JumpTarget, } HasIndirectJump = true; - MBasicBlock *TargetBB = getOrCreateIndirectJumpBB(); + MBasicBlock *TargetBB = getOrCreateIndirectJumpBB(CurrentBlockPC); createInstruction(true, &(Ctx.VoidType), JumpTarget, JumpTargetVar->getVarIdx()); createInstruction(true, Ctx, TargetBB); @@ -1215,6 +1503,29 @@ void EVMMirBuilder::handleJumpI(Operand Dest, Operand Cond) { FallThroughBB); addUniqueSuccessor(InvalidJumpBB); addSuccessor(FallThroughBB); + } else if (Dest.isConstant()) { + const auto &ConstValue = Dest.getConstValue(); + if ((ConstValue[3] | ConstValue[2] | ConstValue[1]) != 0) { + createInstruction(true, Ctx, IsNonZero, InvalidJumpBB, + FallThroughBB); + addUniqueSuccessor(InvalidJumpBB); + addSuccessor(FallThroughBB); + } else { + uint64_t ConstDest = ConstValue[0]; + auto JumpIt = JumpDestTable.find(ConstDest); + if (JumpIt == JumpDestTable.end()) { + createInstruction(true, Ctx, IsNonZero, InvalidJumpBB, + FallThroughBB); + addUniqueSuccessor(InvalidJumpBB); + addSuccessor(FallThroughBB); + } else { + registerPhiIncomingBlock(ConstDest, CurrentBlockPC, CurBB); + createInstruction(true, Ctx, IsNonZero, JumpIt->second, + FallThroughBB); + addSuccessor(JumpIt->second); + addSuccessor(FallThroughBB); + } + } } else { MBasicBlock *JumpTableBB = createBasicBlock(); createInstruction(true, Ctx, IsNonZero, JumpTableBB, @@ -1222,31 +1533,19 @@ void EVMMirBuilder::handleJumpI(Operand Dest, Operand Cond) { addSuccessor(JumpTableBB); addSuccessor(FallThroughBB); setInsertBlock(JumpTableBB); - if (Dest.isConstant()) { - const auto &ConstValue = Dest.getConstValue(); - if ((ConstValue[3] | ConstValue[2] | ConstValue[1]) != 0) { - createInstruction(true, Ctx, InvalidJumpBB); - addSuccessor(InvalidJumpBB); - } else { - uint64_t ConstDest = ConstValue[0]; - implementConstantJump(ConstDest, InvalidJumpBB); - } - } else { - MInstruction *HighOr = createInstruction( - false, OP_or, MirI64Type, DestComponents[1], DestComponents[2]); - HighOr = createInstruction(false, OP_or, MirI64Type, - HighOr, DestComponents[3]); - MInstruction *HighNonZero = createInstruction( - false, CmpInstruction::Predicate::ICMP_NE, &Ctx.I64Type, HighOr, - Zero); - MBasicBlock *ValidJumpBB = createBasicBlock(); - createInstruction(true, Ctx, HighNonZero, InvalidJumpBB, - ValidJumpBB); - addSuccessor(InvalidJumpBB); - addSuccessor(ValidJumpBB); - setInsertBlock(ValidJumpBB); - implementIndirectJump(JumpTarget, InvalidJumpBB); - } + MInstruction *HighOr = createInstruction( + false, OP_or, MirI64Type, DestComponents[1], DestComponents[2]); + HighOr = createInstruction(false, OP_or, MirI64Type, + HighOr, DestComponents[3]); + MInstruction *HighNonZero = createInstruction( + false, CmpInstruction::Predicate::ICMP_NE, &Ctx.I64Type, HighOr, Zero); + MBasicBlock *ValidJumpBB = createBasicBlock(); + createInstruction(true, Ctx, HighNonZero, InvalidJumpBB, + ValidJumpBB); + addSuccessor(InvalidJumpBB); + addSuccessor(ValidJumpBB); + setInsertBlock(ValidJumpBB); + implementIndirectJump(JumpTarget, InvalidJumpBB); } setInsertBlock(FallThroughBB); @@ -1266,11 +1565,13 @@ void EVMMirBuilder::handleJumpDest(const uint64_t &PC) { } if (CurBB != DestBB && !IsExceptionSetBB) { if (CurBB->empty()) { + registerPhiIncomingBlock(PC, CurrentBlockPC, CurBB); CurBB->addSuccessor(DestBB); createInstruction(true, Ctx, DestBB); } else { MInstruction *LastInst = *std::prev(CurBB->end()); if (!LastInst->isTerminator()) { + registerPhiIncomingBlock(PC, CurrentBlockPC, CurBB); CurBB->addSuccessor(DestBB); createInstruction(true, Ctx, DestBB); } @@ -3156,6 +3457,9 @@ void EVMMirBuilder::handleTStore(Operand Index, Operand ValueComponents) { void EVMMirBuilder::handleSelfDestruct(Operand Beneficiary) { const auto &RuntimeFunctions = getRuntimeFunctionTable(); #ifdef ZEN_ENABLE_EVM_GAS_REGISTER + // SELFDESTRUCT is a terminating opcode. Flush the live gas register so the + // runtime helper observes the current callee gas and returns the correct + // leftover amount to the caller. syncGasToMemoryFull(); #endif callRuntimeFor(RuntimeFunctions.HandleSelfDestruct, @@ -3317,6 +3621,20 @@ MInstruction *EVMMirBuilder::loadVariable(Variable *Var) { Var->getVarIdx()); } +PhiInstruction *EVMMirBuilder::createPendingPhi(MType *Type, + size_t NumIncoming) { + return createInstruction(true, Type, NumIncoming); +} + +size_t EVMMirBuilder::getPhiIncomingSlot(PhiInstruction *Phi, + uint64_t PredBlockPC) const { + auto PhiIt = PhiIncomingSlotMap.find(Phi); + ZEN_ASSERT(PhiIt != PhiIncomingSlotMap.end()); + auto SlotIt = PhiIt->second.find(PredBlockPC); + ZEN_ASSERT(SlotIt != PhiIt->second.end()); + return SlotIt->second; +} + MInstruction *EVMMirBuilder::protectUnsafeValue(MInstruction *Value, MType *Type) { Variable *ReusableVar = CurFunc->createVariable(Type); @@ -3790,8 +4108,17 @@ EVMMirBuilder::convertOperandToUNInstruction(const Operand &Param) { } } else if (Param.isU256MultiComponent()) { auto Components = Param.getU256Components(); - for (size_t I = 0; I < N; ++I) { - Result[I] = Components[I]; + if (Components[0] != nullptr) { + for (size_t I = 0; I < N; ++I) { + Result[I] = Components[I]; + } + } else { + auto Vars = Param.getU256VarComponents(); + for (size_t I = 0; I < N; ++I) { + ZEN_ASSERT(Vars[I] != nullptr); + Result[I] = createInstruction( + false, Vars[I]->getType(), Vars[I]->getVarIdx()); + } } } else if (Param.isConstant()) { const U256Value &U256Value = Param.getConstValue(); diff --git a/src/compiler/evm_frontend/evm_mir_compiler.h b/src/compiler/evm_frontend/evm_mir_compiler.h index 21ac1c42c..76a40ead9 100644 --- a/src/compiler/evm_frontend/evm_mir_compiler.h +++ b/src/compiler/evm_frontend/evm_mir_compiler.h @@ -207,6 +207,20 @@ class EVMMirBuilder final { void stackSet(int32_t IndexFromTop, Operand SetValue); Operand stackGet(int32_t IndexFromTop); + void setTrackedStackDepth(uint32_t Depth); + Operand createStackEntryOperand(); + void assignStackEntryOperand(const Operand &Dest, const Operand &Value); + Operand prepareStackPhiIncoming(const Operand &Value); + void registerCurrentBlockPC(uint64_t BlockPC); + Operand materializeStackMergeOperand( + const std::vector &PredBlockPCs, + const std::vector> &IncomingValues); + void assignStackMergeOperand(const Operand &Dest, uint64_t PredBlockPC, + const Operand &Value); + void spillTrackedStack(const std::vector &TrackedStack); + void + spillTrackedStackPreservingPrefix(const std::vector &TrackedStack, + uint32_t PrefixDepth); // PUSH0: place value 0 on stack // PUSH1-PUSH32: Push N bytes onto stack @@ -471,6 +485,8 @@ class EVMMirBuilder final { Operand loadProtectedAddressFieldAsU256(MInstruction *BasePtr, int32_t Offset); MInstruction *getHostArgScratchPtr(std::size_t ScratchSlot); + PhiInstruction *createPendingPhi(MType *Type, size_t NumIncoming); + size_t getPhiIncomingSlot(PhiInstruction *Phi, uint64_t PredBlockPC) const; template T *createInstruction(bool IsStmt, Arguments &&...Args) { @@ -636,7 +652,20 @@ class EVMMirBuilder final { template U256Inst convertOperandToUNInstruction(const Operand &Param); - MBasicBlock *getOrCreateIndirectJumpBB(); + MBasicBlock *getOrCreateIndirectJumpBB(uint64_t SourceBlockPC); + void registerPhiIncomingBlock(uint64_t TargetBlockPC, uint64_t PredBlockPC, + MBasicBlock *PredBB); + void registerDynamicJumpPhiIncomingBlock(uint64_t TargetBlockPC, + uint64_t PredBlockPC, + MBasicBlock *PredBB); + MBasicBlock *getPhiIncomingBlock(uint64_t TargetBlockPC, + uint64_t PredBlockPC) const; + uint64_t getCanonicalJumpDestPC(uint64_t TargetBlockPC) const; + MBasicBlock *resolvePhiIncomingPredecessorBB(uint64_t TargetBlockPC, + MBasicBlock *DirectPredBB) const; + MBasicBlock * + resolveReachablePhiIncomingPredecessorBB(uint64_t TargetBlockPC, + MBasicBlock *CandidateBB) const; CompilerContext &Ctx; MFunction *CurFunc = nullptr; @@ -659,6 +688,7 @@ class EVMMirBuilder final { // Entry blocks for jump targets (may be tiny thunks for shared JUMPDEST // bodies). std::map JumpDestTable; + std::map JumpDestCanonicalPCTable; // Canonical execution blocks for JUMPDEST opcodes in linear decode. std::map JumpDestBodyTable; // Cached skipped-metering for merged consecutive JUMPDEST runs. @@ -671,7 +701,7 @@ class EVMMirBuilder final { std::map> JumpHashReverse; uint64_t HashMask = 0; Variable *JumpTargetVar = nullptr; - MBasicBlock *IndirectJumpBB = nullptr; + std::map IndirectJumpBBs; // Stack check block for stack overflow/underflow checking MBasicBlock *StackCheckBB = nullptr; @@ -679,6 +709,12 @@ class EVMMirBuilder final { Variable *StackSizeVar = nullptr; Variable *MemoryBaseVar = nullptr; Variable *MemorySizeVar = nullptr; + uint64_t CurrentBlockPC = 0; + std::map BlockEntryTable; + std::map> + DynamicPhiIncomingBlockTable; + std::map> PhiIncomingSlotMap; + std::map StackMergePhiVarMap; // Helper methods for memory operations MInstruction *getMemoryDataPointer(); diff --git a/src/compiler/mir/instruction.h b/src/compiler/mir/instruction.h index 818574379..dca46eccd 100644 --- a/src/compiler/mir/instruction.h +++ b/src/compiler/mir/instruction.h @@ -36,6 +36,7 @@ class MInstruction : public NonCopyable { EVM_U256_MUL_RESULT, //===---------- Statement Instructions ----------===// + PHI, DASSIGN, STORE, BR, diff --git a/src/compiler/mir/instructions.cpp b/src/compiler/mir/instructions.cpp index 442039c8f..98ec296f0 100644 --- a/src/compiler/mir/instructions.cpp +++ b/src/compiler/mir/instructions.cpp @@ -60,6 +60,30 @@ void MInstruction::print(llvm::raw_ostream &OS) const { << getOperand<1>() << ", " << getOperand<2>() << ", " << getOperand<3>() << ')'; break; + case PHI: { + auto *phi = llvm::cast(this); + OS << "phi ["; + for (size_t I = 0; I < phi->getNumIncoming(); ++I) { + if (I != 0) { + OS << ", "; + } + MBasicBlock *IncomingBB = phi->getIncomingBlock(I); + const MInstruction *IncomingValue = phi->getIncomingValue(I); + if (IncomingBB) { + OS << '@' << IncomingBB->getIdx(); + } else { + OS << ""; + } + OS << ": "; + if (IncomingValue) { + OS << IncomingValue; + } else { + OS << ""; + } + } + OS << "]\n"; + break; + } case DASSIGN: { auto *assign = llvm::cast(this); OS << '$' << assign->getVarIdx() << " = " << getOperand<0>() << "\n"; diff --git a/src/compiler/mir/instructions.h b/src/compiler/mir/instructions.h index 48dc45f54..6f6ebfebd 100644 --- a/src/compiler/mir/instructions.h +++ b/src/compiler/mir/instructions.h @@ -161,6 +161,71 @@ class NaryInstruction : public FixedOperandInstruction<0> { : FixedOperandInstruction(kind, opcode, 0, type) {} }; +class PhiInstruction : public DynamicOperandInstruction { +public: + using Incoming = std::pair; + + static PhiInstruction *create(CompileMemPool &MemPool, MType *Type, + size_t NumIncoming) { + return DynamicOperandInstruction::createWithMemPool( + MemPool, NumIncoming, Type, NumIncoming); + } + + static PhiInstruction *create(CompileMemPool &MemPool, MType *Type, + llvm::ArrayRef Incomings) { + return DynamicOperandInstruction::createWithMemPool( + MemPool, Incomings.size(), Type, Incomings); + } + + static bool classof(const MInstruction *Inst) { + return Inst->getOpcode() == OP_phi; + } + + size_t getNumIncoming() const { return getNumOperands(); } + + MBasicBlock *getIncomingBlock(size_t Index) const { + ZEN_ASSERT(Index < Blocks.size()); + return Blocks[Index]; + } + + const MInstruction *getIncomingValue(size_t Index) const { + ZEN_ASSERT(Index < getNumOperands()); + return getOperand(static_cast(Index)); + } + + void setIncoming(size_t Index, MBasicBlock *Block, MInstruction *Value) { + ZEN_ASSERT(Index < Blocks.size()); + Blocks[Index] = Block; + if (Value != nullptr) { + setOperand(static_cast(Index), Value); + } else { + getOperand(static_cast(Index)) = nullptr; + } + } + +private: + friend class DynamicOperandInstruction; + + PhiInstruction(CompileMemPool &MemPool, MType *Type, size_t NumIncoming) + : DynamicOperandInstruction(MInstruction::PHI, OP_phi, NumIncoming, Type), + Blocks(NumIncoming, MemPool) { + for (size_t Index = 0; Index < NumIncoming; ++Index) { + Blocks[Index] = nullptr; + getOperand(static_cast(Index)) = nullptr; + } + } + + PhiInstruction(CompileMemPool &MemPool, MType *Type, + llvm::ArrayRef Incomings) + : PhiInstruction(MemPool, Type, Incomings.size()) { + for (size_t Index = 0; Index < Incomings.size(); ++Index) { + setIncoming(Index, Incomings[Index].first, Incomings[Index].second); + } + } + + CompileVector Blocks; +}; + class DassignInstruction : public UnaryInstruction { public: template diff --git a/src/compiler/mir/opcode.h b/src/compiler/mir/opcode.h index 794a3d6bc..b5a67ed95 100644 --- a/src/compiler/mir/opcode.h +++ b/src/compiler/mir/opcode.h @@ -34,7 +34,7 @@ enum Opcode : uint16_t { OP_CTRL_STMT_START = OP_br, OP_CTRL_STMT_END = OP_return, - OP_OTHER_STMT_START = OP_dassign, + OP_OTHER_STMT_START = OP_phi, OP_OTHER_STMT_END = OP_wasm_check_stack_boundary, OP_START = OP_UNARY_EXPR_START, @@ -49,6 +49,7 @@ class OpcodeDesc { // TODO: use desc information switch (opcode) { case OP_dassign: + case OP_phi: case OP_return: case OP_store: return true; diff --git a/src/compiler/mir/opcodes.def b/src/compiler/mir/opcodes.def index 0f030d06c..2151537d5 100644 --- a/src/compiler/mir/opcodes.def +++ b/src/compiler/mir/opcodes.def @@ -2,7 +2,7 @@ #define OPCODE(X) #endif -OPCODE(clz) // OP_UNARY_EXPR_START, OP_START +OPCODE(clz) // OP_UNARY_EXPR_START, OP_START OPCODE(ctz) OPCODE(not) OPCODE(popcnt) @@ -13,9 +13,9 @@ OPCODE(fpsqrt) OPCODE(fpround_ceil) OPCODE(fpround_floor) OPCODE(fpround_trunc) -OPCODE(fpround_nearest) // OP_UNARY_EXPR_END +OPCODE(fpround_nearest) // OP_UNARY_EXPR_END -OPCODE(add) // OP_BIN_EXPR_START +OPCODE(add) // OP_BIN_EXPR_START OPCODE(sub) OPCODE(mul) OPCODE(sdiv) @@ -34,14 +34,14 @@ OPCODE(fpdiv) OPCODE(fpmin) OPCODE(fpmax) OPCODE(fpcopysign) -OPCODE(wasm_sadd_overflow) // OP_OVERFLOW_BIN_EXPR_START +OPCODE(wasm_sadd_overflow) // OP_OVERFLOW_BIN_EXPR_START OPCODE(wasm_uadd_overflow) OPCODE(wasm_ssub_overflow) OPCODE(wasm_usub_overflow) OPCODE(wasm_smul_overflow) -OPCODE(wasm_umul_overflow) // OP_OVERFLOW_BIN_EXPR_END, OP_BIN_EXPR_END +OPCODE(wasm_umul_overflow) // OP_OVERFLOW_BIN_EXPR_END, OP_BIN_EXPR_END -OPCODE(inttoptr) // OP_CONV_EXPR_START +OPCODE(inttoptr) // OP_CONV_EXPR_START OPCODE(ptrtoint) OPCODE(trunc) OPCODE(sext) @@ -52,9 +52,10 @@ OPCODE(sitofp) OPCODE(uitofp) OPCODE(bitcast) OPCODE(wasm_fptosi) -OPCODE(wasm_fptoui) // OP_CONV_EXPR_END +OPCODE(wasm_fptoui) // OP_CONV_EXPR_END -OPCODE(dread) // OP_OTHER_EXPR_START +OPCODE(phi) // OP_OTHER_EXPR_START +OPCODE(dread) OPCODE(const) OPCODE(cmp) OPCODE(adc) @@ -65,21 +66,21 @@ OPCODE(wasm_sadd128_overflow) OPCODE(wasm_uadd128_overflow) OPCODE(wasm_ssub128_overflow) OPCODE(wasm_usub128_overflow) -OPCODE(evm_umul128_lo) // 64x64->64 multiplication (low bits) -OPCODE(evm_umul128_hi) // extract high 64 bits from evm_umul128_lo -OPCODE(evm_u256_mul) // 256x256->256 multiplication pseudo op -OPCODE(evm_u256_mul_result) // extract extra limb from evm_u256_mul - // OP_OTHER_EXPR_END +OPCODE(evm_umul128_lo) // 64x64->64 multiplication (low bits) +OPCODE(evm_umul128_hi) // extract high 64 bits from evm_umul128_lo +OPCODE(evm_u256_mul) // 256x256->256 multiplication pseudo op +OPCODE(evm_u256_mul_result) // extract extra limb from evm_u256_mul + // OP_OTHER_EXPR_END -OPCODE(br) // OP_CTRL_STMT_START +OPCODE(br) // OP_CTRL_STMT_START OPCODE(br_if) OPCODE(switch) OPCODE(call) OPCODE(icall) -OPCODE(return) // OP_CTRL_STMT_END +OPCODE(return) // OP_CTRL_STMT_END -OPCODE(dassign) // OP_OTHER_STMT_START +OPCODE(dassign) // OP_OTHER_STMT_START OPCODE(store) OPCODE(wasm_check_memory_access) OPCODE(wasm_visit_stack_guard) -OPCODE(wasm_check_stack_boundary) // OP_OTHER_STMT_END, OP_END +OPCODE(wasm_check_stack_boundary) // OP_OTHER_STMT_END, OP_END diff --git a/src/compiler/mir/pass/verifier.cpp b/src/compiler/mir/pass/verifier.cpp index ec01ef031..d0e421af2 100644 --- a/src/compiler/mir/pass/verifier.cpp +++ b/src/compiler/mir/pass/verifier.cpp @@ -165,6 +165,50 @@ void MVerifier::visitSelectInstruction(SelectInstruction &I) { MVisitor::visitSelectInstruction(I); } +void MVerifier::visitPhiInstruction(PhiInstruction &I) { + CHECK(!I.getType()->isVoid(), "The type of phi instruction must not be void"); + MBasicBlock *PhiBB = I.getBasicBlock(); + CHECK(PhiBB != nullptr, "phi instruction must belong to a basic block"); + auto PredRange = PhiBB->predecessors(); + if (I.getNumIncoming() != + static_cast(std::distance(PredRange.begin(), PredRange.end()))) { + OS << "[phi-verifier-debug] current block dump\n"; + PhiBB->print(OS); + for (MBasicBlock *PredBB : PredRange) { + OS << "[phi-verifier-debug] predecessor block dump\n"; + PredBB->print(OS); + } + } + CHECK(I.getNumIncoming() == static_cast(std::distance( + PredRange.begin(), PredRange.end())), + "The number of phi incoming values must match predecessor count"); + + CompileSet SeenPreds(CurFunc->getContext().ThreadMemPool); + for (size_t Index = 0; Index < I.getNumIncoming(); ++Index) { + MBasicBlock *IncomingBB = I.getIncomingBlock(Index); + const MInstruction *IncomingValue = I.getIncomingValue(Index); + CHECK(IncomingBB != nullptr, + "phi incoming block must be populated before verification"); + CHECK(IncomingValue != nullptr, + "phi incoming value must be populated before verification"); + auto PredIt = std::find(PredRange.begin(), PredRange.end(), IncomingBB); + if (PredIt == PredRange.end()) { + OS << "[phi-verifier-debug] current block dump\n"; + PhiBB->print(OS); + if (IncomingBB) { + OS << "[phi-verifier-debug] incoming block dump\n"; + IncomingBB->print(OS); + } + } + CHECK(PredIt != PredRange.end(), + "phi incoming block must be a predecessor of the current block"); + CHECK(SeenPreds.insert(IncomingBB).second, + "phi incoming block must appear at most once"); + CHECK(IncomingValue->getType()->getKind() == I.getType()->getKind(), + "phi incoming value type must match phi result type"); + } +} + void MVerifier::visitDassignInstruction(DassignInstruction &I) { MType *Type = I.getType(); CHECK( diff --git a/src/compiler/mir/pass/verifier.h b/src/compiler/mir/pass/verifier.h index 622068008..21358a25d 100644 --- a/src/compiler/mir/pass/verifier.h +++ b/src/compiler/mir/pass/verifier.h @@ -28,6 +28,16 @@ class MVerifier final : public MVisitor { if (BB.empty()) { return; } + bool SeenNonPhi = false; + for (const MInstruction *Inst : BB) { + if (Inst->getKind() == MInstruction::PHI) { + CHECK(!SeenNonPhi, "phi instructions in BB @" + + std::to_string(BB.getIdx()) + + " must be contiguous at block start"); + } else { + SeenNonPhi = true; + } + } const MInstruction *LastInst = *std::prev(BB.end()); CHECK(LastInst->isTerminator(), "The last instruction in BB @" + std::to_string(BB.getIdx()) + @@ -47,6 +57,7 @@ class MVerifier final : public MVisitor { void visitSbbInstruction(SbbInstruction &I) override; void visitCmpInstruction(CmpInstruction &I) override; void visitSelectInstruction(SelectInstruction &I) override; + void visitPhiInstruction(PhiInstruction &I) override; void visitDassignInstruction(DassignInstruction &I) override; void visitLoadInstruction(LoadInstruction &I) override; void visitStoreInstruction(StoreInstruction &I) override; diff --git a/src/compiler/mir/pass/visitor.h b/src/compiler/mir/pass/visitor.h index 4721eff4b..e0a8fc205 100644 --- a/src/compiler/mir/pass/visitor.h +++ b/src/compiler/mir/pass/visitor.h @@ -67,6 +67,9 @@ class MVisitor { case MInstruction::SELECT: visitSelectInstruction(static_cast(I)); break; + case MInstruction::PHI: + visitPhiInstruction(static_cast(I)); + break; case MInstruction::DASSIGN: visitDassignInstruction(static_cast(I)); break; @@ -144,6 +147,7 @@ class MVisitor { virtual void visitAdcInstruction(AdcInstruction &I) { VISIT_OPERAND_3 } virtual void visitSbbInstruction(SbbInstruction &I) { VISIT_OPERAND_3 } virtual void visitSelectInstruction(SelectInstruction &I) { VISIT_OPERAND_3 } + virtual void visitPhiInstruction(PhiInstruction &I) {} virtual void visitDassignInstruction(DassignInstruction &I) { VISIT_OPERAND_1 } diff --git a/src/evm/opcode_handlers.cpp b/src/evm/opcode_handlers.cpp index cad5226d5..3f9b7ead3 100644 --- a/src/evm/opcode_handlers.cpp +++ b/src/evm/opcode_handlers.cpp @@ -10,6 +10,9 @@ #include "host/evm/crypto.h" #include "runtime/evm_instance.h" +#include +#include + thread_local zen::evm::EVMFrame *zen::evm::EVMResource::CurrentFrame = nullptr; thread_local zen::evm::InterpreterExecContext *zen::evm::EVMResource::CurrentContext = nullptr; @@ -22,6 +25,8 @@ using namespace zen::runtime; namespace { +bool chargeGas(EVMFrame *Frame, uint64_t GasCost); + evmc_revision currentRevision() { auto *Context = EVMResource::getInterpreterExecContext(); if (!Context) { @@ -31,6 +36,53 @@ evmc_revision currentRevision() { return Instance ? Instance->getRevision() : DEFAULT_REVISION; } +constexpr uint8_t DelegationMagicBytes[] = {0xef, 0x01, 0x00}; + +bool resolveDelegatedCallCodeAddress(EVMFrame *Frame, evmc::address Dest, + evmc::address &CodeAddress) { + CodeAddress = Dest; + if (currentRevision() < EVMC_PRAGUE) { + return true; + } + + uint8_t Designation[sizeof(DelegationMagicBytes) + sizeof(evmc::address)] = + {}; + const size_t Copied = + Frame->Host->copy_code(Dest, 0, Designation, sizeof(Designation)); + if (Copied < sizeof(DelegationMagicBytes) || + std::memcmp(Designation, DelegationMagicBytes, + sizeof(DelegationMagicBytes)) != 0) { + return true; + } + + if (Copied != sizeof(Designation)) { + return true; + } + + std::memcpy(CodeAddress.bytes, Designation + sizeof(DelegationMagicBytes), + sizeof(CodeAddress.bytes)); + const uint64_t DelegateAccessCost = + Frame->Host->access_account(CodeAddress) == EVMC_ACCESS_COLD + ? COLD_ACCOUNT_ACCESS_COST + : WARM_STORAGE_READ_COST; + std::fprintf(stderr, + "delegation interp hit rev=%d copied=%zu cost=%llu " + "dest=%02x%02x code=%02x%02x gas=%lld\n", + static_cast(currentRevision()), Copied, + static_cast(DelegateAccessCost), + Dest.bytes[0], Dest.bytes[1], CodeAddress.bytes[0], + CodeAddress.bytes[1], static_cast(Frame->Msg.gas)); + if (static_cast(Frame->Msg.gas) < DelegateAccessCost) { + auto *Context = EVMResource::getInterpreterExecContext(); + if (Context) { + Context->setStatus(EVMC_OUT_OF_GAS); + } + return false; + } + Frame->Msg.gas -= static_cast(DelegateAccessCost); + return true; +} + } // namespace /* ---------- Define gas cost macros begin ---------- */ @@ -1251,6 +1303,11 @@ void CallHandler::doExecute() { } } + evmc::address CodeAddress = Dest; + if (!resolveDelegatedCallCodeAddress(Frame, Dest, CodeAddress)) { + return; + } + if (OpCode == evmc_opcode::OP_CALL && HasValue && Frame->isStaticMode()) { Context->setStatus(EVMC_STATIC_MODE_VIOLATION); return; @@ -1355,10 +1412,15 @@ void CallHandler::doExecute() { ? Frame->Msg.value : intx::be::store(Value), .create2_salt = {}, - .code_address = Dest, + .code_address = CodeAddress, .code = nullptr, .code_size = 0, }; + if (std::memcmp(Dest.bytes, CodeAddress.bytes, sizeof(Dest.bytes)) != 0) { + NewMsg.flags |= EVMC_DELEGATED; + } else { + NewMsg.flags &= ~uint32_t(EVMC_DELEGATED); + } const auto Result = Frame->Host->call(NewMsg); Context->setResource(); diff --git a/src/runtime/evm_module.cpp b/src/runtime/evm_module.cpp index 673054110..33cbf079a 100644 --- a/src/runtime/evm_module.cpp +++ b/src/runtime/evm_module.cpp @@ -15,12 +15,41 @@ #include #include +#ifdef ZEN_ENABLE_JIT_PRECOMPILE_FALLBACK +#include "compiler/evm_frontend/evm_analyzer.h" +#endif + #ifdef ZEN_ENABLE_MULTIPASS_JIT #include "compiler/evm_compiler.h" #endif namespace zen::runtime { +#ifdef ZEN_ENABLE_JIT_PRECOMPILE_FALLBACK +namespace { + +bool hasUnresolvedCompatibleDynamicReturnTrampoline( + const COMPILER::EVMAnalyzer &Analyzer) { + for (const auto &[EntryPC, Info] : Analyzer.getBlockInfos()) { + if (!Info.HasDynamicJump) { + continue; + } + if (Analyzer.getOutgoingCompatibleDynamicJumpShapeClassForBlock(EntryPC) == + 0) { + continue; + } + if (!Analyzer + .canTransferCompatibleDynamicJumpTargetsWithoutRuntimeMaterialization( + EntryPC)) { + return true; + } + } + return false; +} + +} // namespace +#endif + EVMModule::EVMModule(Runtime *RT) : BaseModule(RT, ModuleType::EVM), Code(nullptr), CodeSize(0) { // do nothing @@ -65,7 +94,18 @@ EVMModuleUniquePtr EVMModule::newEVMModule(Runtime &RT, ZEN_ASSERT(RT.getEVMHost()); Mod->Host = RT.getEVMHost(); +#ifdef ZEN_ENABLE_JIT_PRECOMPILE_FALLBACK + if (RT.getConfig().Mode == common::RunMode::MultipassMode) { + (void)Mod->shouldInterpretInMultipass(); + } +#endif + if (RT.getConfig().Mode != common::RunMode::InterpMode) { +#ifdef ZEN_ENABLE_JIT_PRECOMPILE_FALLBACK + if (Mod->shouldInterpretInMultipass()) { + return Mod; + } +#endif action::performEVMJITCompile(*Mod); } @@ -84,4 +124,23 @@ void EVMModule::initBytecodeCache() const { evm::buildBytecodeCache(BytecodeCache, Code, CodeSize, Revision); } +#ifdef ZEN_ENABLE_JIT_PRECOMPILE_FALLBACK +bool EVMModule::shouldInterpretInMultipass() const { + if (!MultipassFallbackInfoInitialized) { + initializeMultipassFallbackInfo(); + } + return ShouldInterpretInMultipass; +} + +void EVMModule::initializeMultipassFallbackInfo() const { + COMPILER::EVMAnalyzer Analyzer(Revision); + Analyzer.analyze(reinterpret_cast(Code), CodeSize); + const auto &JITResult = Analyzer.getJITSuitability(); + ShouldInterpretInMultipass = + JITResult.ShouldFallback || + hasUnresolvedCompatibleDynamicReturnTrampoline(Analyzer); + MultipassFallbackInfoInitialized = true; +} +#endif + } // namespace zen::runtime diff --git a/src/runtime/evm_module.h b/src/runtime/evm_module.h index 43094717e..02ba1ed77 100644 --- a/src/runtime/evm_module.h +++ b/src/runtime/evm_module.h @@ -44,6 +44,10 @@ class EVMModule final : public BaseModule { return static_cast(offsetof(EVMModule, CodeSize)); } +#ifdef ZEN_ENABLE_JIT_PRECOMPILE_FALLBACK + bool shouldInterpretInMultipass() const; +#endif + #ifdef ZEN_ENABLE_JIT common::CodeMemPool &getJITCodeMemPool() { return JITCodeMemPool; } @@ -70,6 +74,12 @@ class EVMModule final : public BaseModule { mutable evm::EVMBytecodeCache BytecodeCache; evmc_revision Revision = zen::evm::DEFAULT_REVISION; +#ifdef ZEN_ENABLE_JIT_PRECOMPILE_FALLBACK + void initializeMultipassFallbackInfo() const; + mutable bool MultipassFallbackInfoInitialized = false; + mutable bool ShouldInterpretInMultipass = false; +#endif + #ifdef ZEN_ENABLE_JIT common::CodeMemPool JITCodeMemPool; void *JITCode = nullptr; diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 89a1c2f37..5d28aa60d 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -60,6 +60,9 @@ if(ZEN_ENABLE_SPEC_TEST) if(ZEN_ENABLE_EVM) add_subdirectory(mpt) add_executable(evmInterpTests evm_interp_tests.cpp) + if(ZEN_ENABLE_MULTIPASS_JIT) + add_executable(evmJitFrontendTests evm_jit_frontend_tests.cpp) + endif() add_executable( solidityContractTests evm_precompiles.hpp solidity_contract_tests.cpp solidity_test_helpers.cpp @@ -95,6 +98,9 @@ if(ZEN_ENABLE_SPEC_TEST) if(ZEN_ENABLE_EVM) target_compile_options(evmInterpTests PRIVATE -fsanitize=address) + if(ZEN_ENABLE_MULTIPASS_JIT) + target_compile_options(evmJitFrontendTests PRIVATE -fsanitize=address) + endif() target_compile_options(solidityContractTests PRIVATE -fsanitize=address) target_compile_options(mptCompareCpp PRIVATE -fsanitize=address) endif() @@ -117,6 +123,13 @@ if(ZEN_ENABLE_SPEC_TEST) PRIVATE dtvmcore rapidjson yaml-cpp gtest_main -fsanitize=address PUBLIC ${GTEST_BOTH_LIBRARIES} ) + if(ZEN_ENABLE_MULTIPASS_JIT) + target_link_libraries( + evmJitFrontendTests + PRIVATE compiler dtvmcore gtest_main -fsanitize=address + PUBLIC ${GTEST_BOTH_LIBRARIES} + ) + endif() target_link_libraries( solidityContractTests PRIVATE dtvmcore rapidjson mpt gtest_main -fsanitize=address @@ -161,6 +174,14 @@ if(ZEN_ENABLE_SPEC_TEST) -static-libasan PUBLIC ${GTEST_BOTH_LIBRARIES} ) + if(ZEN_ENABLE_MULTIPASS_JIT) + target_link_libraries( + evmJitFrontendTests + PRIVATE compiler dtvmcore gtest_main -fsanitize=address + -static-libasan + PUBLIC ${GTEST_BOTH_LIBRARIES} + ) + endif() target_link_libraries( solidityContractTests PRIVATE dtvmcore @@ -213,6 +234,13 @@ if(ZEN_ENABLE_SPEC_TEST) PRIVATE dtvmcore rapidjson yaml-cpp gtest_main PUBLIC ${GTEST_BOTH_LIBRARIES} ) + if(ZEN_ENABLE_MULTIPASS_JIT) + target_link_libraries( + evmJitFrontendTests + PRIVATE compiler dtvmcore gtest_main + PUBLIC ${GTEST_BOTH_LIBRARIES} + ) + endif() target_link_libraries( solidityContractTests PRIVATE dtvmcore rapidjson mpt gtest_main CLI11::CLI11 @@ -248,6 +276,9 @@ if(ZEN_ENABLE_SPEC_TEST) if(ZEN_ENABLE_EVM) add_test(NAME evmInterpTests COMMAND evmInterpTests) + if(ZEN_ENABLE_MULTIPASS_JIT) + add_test(NAME evmJitFrontendTests COMMAND evmJitFrontendTests) + endif() add_test( NAME solidityContractTests COMMAND diff --git a/src/tests/evm_jit_frontend_tests.cpp b/src/tests/evm_jit_frontend_tests.cpp new file mode 100644 index 000000000..f48350b45 --- /dev/null +++ b/src/tests/evm_jit_frontend_tests.cpp @@ -0,0 +1,738 @@ +// Copyright (C) 2025 the DTVM authors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "action/evm_bytecode_visitor.h" +#include "compiler/evm_frontend/evm_analyzer.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace { + +using COMPILER::EVMAnalyzer; + +EVMAnalyzer analyzeBytecode(const std::vector &Bytecode) { + EVMAnalyzer Analyzer(EVMC_CANCUN); + const uint8_t *Data = Bytecode.empty() ? nullptr : Bytecode.data(); + Analyzer.analyze(Data, Bytecode.size()); + return Analyzer; +} + +EVMAnalyzer +analyzeSuitabilityOnlyBytecode(const std::vector &Bytecode) { + EVMAnalyzer Analyzer(EVMC_CANCUN); + const uint8_t *Data = Bytecode.empty() ? nullptr : Bytecode.data(); + Analyzer.analyzeSuitabilityOnly(Data, Bytecode.size()); + return Analyzer; +} + +const EVMAnalyzer::BlockInfo *findBlock(const EVMAnalyzer &Analyzer, + uint64_t EntryPC) { + const auto &Blocks = Analyzer.getBlockInfos(); + auto It = Blocks.find(EntryPC); + if (It == Blocks.end()) { + return nullptr; + } + return &It->second; +} + +void expectPCList(const std::vector &Actual, + std::initializer_list Expected) { + ASSERT_EQ(Actual.size(), Expected.size()); + size_t Index = 0; + for (uint64_t ExpectedPC : Expected) { + EXPECT_EQ(Actual[Index], ExpectedPC) << "mismatch at index " << Index; + ++Index; + } +} + +struct MockOperand { + using U256Value = std::array; + + MockOperand() = default; + explicit MockOperand(uint64_t Low) : Value{Low, 0, 0, 0}, Constant(true) {} + explicit MockOperand(std::shared_ptr Slot) + : Slot(std::move(Slot)) {} + + bool isConstant() const { return Constant; } + + const U256Value &getConstValue() const { + ZEN_ASSERT(Constant && "mock operand must be constant"); + return Value; + } + + U256Value resolvedValue() const { + if (Slot) { + return *Slot; + } + return Value; + } + + void assign(const MockOperand &Other) { + ZEN_ASSERT(Slot && "mock operand slot missing"); + *Slot = Other.resolvedValue(); + } + +private: + U256Value Value = {0, 0, 0, 0}; + bool Constant = false; + std::shared_ptr Slot; +}; + +struct MockStackAccessStats { + uint32_t StackPopCount = 0; + uint32_t StackPushCount = 0; + uint32_t StackGetCount = 0; + uint32_t StackSetCount = 0; +}; + +class MockEVMBuilder { +public: + using CompilerContext = COMPILER::EVMFrontendContext; + using Operand = MockOperand; + +#define MOCK_OPERAND_STUB(Name) \ + template Operand Name(Args...) { return Operand(0); } + +#define MOCK_VOID_STUB(Name) \ + template void Name(Args...) {} + + void initEVM(CompilerContext *) { + CurrentOpcode = 0xff; + Trapped = false; + Undefined = false; + } + + void finalizeEVMBase() {} + + bool isOpcodeDefined(evmc_opcode Opcode) const { + const auto *InstructionNames = + evmc_get_instruction_names_table(EVMC_CANCUN); + return InstructionNames && InstructionNames[Opcode] != nullptr; + } + + void meterOpcode(evmc_opcode Opcode, uint64_t) { + CurrentOpcode = static_cast(Opcode); + } + + void meterOpcodeRange(uint64_t, uint64_t) {} + + void enableRuntimeStackChecks() { EnableRuntimeStackChecks = true; } + + void createStackCheckBlock(int32_t MinSize, int32_t MaxSize) { + if (!EnableRuntimeStackChecks) { + return; + } + if (RuntimeStack.size() < static_cast(std::max(MinSize, 0))) { + Trapped = true; + return; + } + if (RuntimeStack.size() > static_cast(std::max(MaxSize, 0))) { + Trapped = true; + } + } + + Operand handlePush(const zen::common::Bytes &Data) { + uint64_t Low = 0; + for (zen::common::Byte Byte : Data) { + Low = (Low << 8) | static_cast(std::to_integer(Byte)); + } + LastPushValue = {Low, 0, 0, 0}; + HasLastPushValue = true; + return Operand(Low); + } + + void stackPush(Operand PushValue) { + Stats[CurrentOpcode].StackPushCount++; + RuntimeStack.push_back(PushValue); + } + + Operand stackPop() { + Stats[CurrentOpcode].StackPopCount++; + ZEN_ASSERT(!RuntimeStack.empty() && "mock runtime stack underflow"); + Operand Top = RuntimeStack.back(); + RuntimeStack.pop_back(); + return Top; + } + + void stackSet(int32_t IndexFromTop, Operand SetValue) { + Stats[CurrentOpcode].StackSetCount++; + size_t Index = RuntimeStack.size() - static_cast(IndexFromTop) - 1; + RuntimeStack[Index] = SetValue; + } + + Operand stackGet(int32_t IndexFromTop) { + Stats[CurrentOpcode].StackGetCount++; + size_t Index = RuntimeStack.size() - static_cast(IndexFromTop) - 1; + return RuntimeStack[Index]; + } + + void setTrackedStackDepth(uint32_t Depth) { + if (RuntimeStack.size() > Depth) { + RuntimeStack.resize(Depth); + } + } + + Operand createStackEntryOperand() { + return Operand(std::make_shared( + MockOperand::U256Value{0, 0, 0, 0})); + } + + void assignStackEntryOperand(const Operand &Dest, const Operand &Value) { + Operand Copy = Dest; + Copy.assign(Value); + } + + void spillTrackedStack(const std::vector &TrackedStack) { + RuntimeStack = TrackedStack; + } + + void setCurrentDebugBlockPC(uint64_t) {} + + template + Operand handleBinaryArithmetic(Operand, Operand) { + return Operand(0); + } + + template + Operand handleCompareOp(Operand, Operand) { + return Operand(0); + } + + template + Operand handleBitwiseOp(Operand, Operand) { + return Operand(0); + } + + template + Operand handleShift(Operand, Operand) { + return Operand(0); + } + + template + void handleLogWithTopics(Args...) {} + + Operand handleCall(Operand, Operand, Operand, Operand, Operand, Operand, + Operand) { + return Operand(0); + } + + Operand handleCallCode(Operand, Operand, Operand, Operand, Operand, Operand, + Operand) { + return Operand(0); + } + + Operand handleDelegateCall(Operand, Operand, Operand, Operand, Operand, + Operand) { + return Operand(0); + } + + Operand handleStaticCall(Operand, Operand, Operand, Operand, Operand, + Operand) { + return Operand(0); + } + + MOCK_OPERAND_STUB(handleAddMod); + MOCK_OPERAND_STUB(handleAddress); + MOCK_OPERAND_STUB(handleBalance); + MOCK_OPERAND_STUB(handleBaseFee); + MOCK_OPERAND_STUB(handleBlobBaseFee); + MOCK_OPERAND_STUB(handleBlobHash); + MOCK_OPERAND_STUB(handleBlockHash); + MOCK_OPERAND_STUB(handleByte); + MOCK_OPERAND_STUB(handleCallDataLoad); + MOCK_OPERAND_STUB(handleCallDataSize); + MOCK_OPERAND_STUB(handleCallValue); + MOCK_OPERAND_STUB(handleCaller); + MOCK_OPERAND_STUB(handleChainId); + MOCK_OPERAND_STUB(handleClz); + MOCK_OPERAND_STUB(handleCodeSize); + MOCK_OPERAND_STUB(handleCoinBase); + MOCK_OPERAND_STUB(handleCreate); + MOCK_OPERAND_STUB(handleCreate2); + MOCK_OPERAND_STUB(handleDiv); + MOCK_OPERAND_STUB(handleExp); + MOCK_OPERAND_STUB(handleExtCodeHash); + MOCK_OPERAND_STUB(handleExtCodeSize); + MOCK_OPERAND_STUB(handleGas); + MOCK_OPERAND_STUB(handleGasLimit); + MOCK_OPERAND_STUB(handleGasPrice); + MOCK_OPERAND_STUB(handleKeccak256); + MOCK_OPERAND_STUB(handleMLoad); + MOCK_OPERAND_STUB(handleMSize); + MOCK_OPERAND_STUB(handleMod); + MOCK_OPERAND_STUB(handleMul); + MOCK_OPERAND_STUB(handleMulMod); + MOCK_OPERAND_STUB(handleNot); + MOCK_OPERAND_STUB(handleNumber); + MOCK_OPERAND_STUB(handleOrigin); + MOCK_OPERAND_STUB(handlePC); + MOCK_OPERAND_STUB(handlePrevRandao); + MOCK_OPERAND_STUB(handleReturnDataSize); + MOCK_OPERAND_STUB(handleSDiv); + MOCK_OPERAND_STUB(handleSLoad); + MOCK_OPERAND_STUB(handleSMod); + MOCK_OPERAND_STUB(handleSelfBalance); + MOCK_OPERAND_STUB(handleSignextend); + MOCK_OPERAND_STUB(handleTimestamp); + MOCK_OPERAND_STUB(handleTLoad); + + MOCK_VOID_STUB(handleCallDataCopy); + MOCK_VOID_STUB(handleCodeCopy); + MOCK_VOID_STUB(handleExtCodeCopy); + MOCK_VOID_STUB(handleMCopy); + MOCK_VOID_STUB(handleMStore); + MOCK_VOID_STUB(handleMStore8); + MOCK_VOID_STUB(handleReturn); + MOCK_VOID_STUB(handleReturnDataCopy); + MOCK_VOID_STUB(handleRevert); + MOCK_VOID_STUB(handleSStore); + MOCK_VOID_STUB(handleSelfDestruct); + MOCK_VOID_STUB(handleTStore); + + void handleJump(Operand) {} + void handleJumpI(Operand, Operand) {} + void handleJumpDest(const uint64_t &) {} + void handleStop() {} + void handleUndefined() { Undefined = true; } + void handleInvalid() { Undefined = true; } + void handleTrap(zen::common::ErrorCode) { Trapped = true; } + void fallbackToInterpreter(uint64_t) {} + void releaseOperand(Operand) {} + + const MockStackAccessStats &accessStats(evmc_opcode Opcode) const { + return Stats[static_cast(Opcode)]; + } + + bool hasLastPushValue() const { return HasLastPushValue; } + + MockOperand::U256Value lastPushValue() const { + ZEN_ASSERT(HasLastPushValue && "mock push value is missing"); + return LastPushValue; + } + + size_t runtimeStackDepth() const { return RuntimeStack.size(); } + + MockOperand::U256Value topStackValue() const { + ZEN_ASSERT(!RuntimeStack.empty() && "mock runtime stack is empty"); + return RuntimeStack.back().resolvedValue(); + } + + bool Trapped = false; + bool Undefined = false; + +private: + bool EnableRuntimeStackChecks = false; + uint8_t CurrentOpcode = 0xff; + std::array Stats = {}; + std::vector RuntimeStack; + MockOperand::U256Value LastPushValue = {0, 0, 0, 0}; + bool HasLastPushValue = false; + +#undef MOCK_OPERAND_STUB +#undef MOCK_VOID_STUB +}; + +TEST(EVMJITFrontendAnalyzerTest, ConstantJumpCanonicalizesJumpDestRuns) { + const std::vector Bytecode = { + 0x60, 0x04, // PUSH1 0x04 + 0x56, // JUMP + 0x5b, // JUMPDEST + 0x5b, // JUMPDEST + 0x00 // STOP + }; + + const EVMAnalyzer Analyzer = analyzeBytecode(Bytecode); + + EXPECT_TRUE(Analyzer.hasCanonicalJumpDest(3)); + EXPECT_TRUE(Analyzer.hasCanonicalJumpDest(4)); + EXPECT_EQ(Analyzer.getCanonicalJumpDestPC(3), 4U); + EXPECT_EQ(Analyzer.getCanonicalJumpDestPC(4), 4U); + + const auto *EntryBlock = findBlock(Analyzer, 0); + const auto *JumpDestBlock = findBlock(Analyzer, 4); + ASSERT_NE(EntryBlock, nullptr); + ASSERT_NE(JumpDestBlock, nullptr); + EXPECT_EQ(findBlock(Analyzer, 3), nullptr); + + EXPECT_TRUE(EntryBlock->HasConstantJump); + EXPECT_EQ(EntryBlock->ConstantJumpTargetPC, 4U); + expectPCList(EntryBlock->Successors, {4}); + + EXPECT_TRUE(JumpDestBlock->IsJumpDest); + expectPCList(JumpDestBlock->Predecessors, {0}); +} + +TEST(EVMJITFrontendAnalyzerTest, + SuitabilityOnlyKeepsFallbackMetricsWithoutBuildingCfg) { + const std::vector Bytecode = { + 0x60, 0x04, // PUSH1 0x04 + 0x56, // JUMP + 0x5b, // JUMPDEST + 0x5b, // JUMPDEST + 0x00 // STOP + }; + + const EVMAnalyzer FullAnalyzer = analyzeBytecode(Bytecode); + const EVMAnalyzer SuitabilityOnlyAnalyzer = + analyzeSuitabilityOnlyBytecode(Bytecode); + + const auto &Full = FullAnalyzer.getJITSuitability(); + const auto &SuitabilityOnly = SuitabilityOnlyAnalyzer.getJITSuitability(); + + EXPECT_EQ(SuitabilityOnly.ShouldFallback, Full.ShouldFallback); + EXPECT_EQ(SuitabilityOnly.BytecodeSize, Full.BytecodeSize); + EXPECT_EQ(SuitabilityOnly.MirEstimate, Full.MirEstimate); + EXPECT_EQ(SuitabilityOnly.RAExpensiveCount, Full.RAExpensiveCount); + EXPECT_EQ(SuitabilityOnly.MaxConsecutiveExpensive, + Full.MaxConsecutiveExpensive); + EXPECT_EQ(SuitabilityOnly.MaxBlockExpensiveCount, + Full.MaxBlockExpensiveCount); + EXPECT_EQ(SuitabilityOnly.DupFeedbackPatternCount, + Full.DupFeedbackPatternCount); + + EXPECT_TRUE(SuitabilityOnlyAnalyzer.getBlockInfos().empty()); + EXPECT_FALSE(SuitabilityOnlyAnalyzer.hasCanonicalJumpDest(3)); + EXPECT_FALSE(SuitabilityOnlyAnalyzer.hasCanonicalJumpDest(4)); + EXPECT_FALSE(SuitabilityOnlyAnalyzer.hasUnknownDynamicJumpTargets()); +} + +TEST(EVMJITFrontendAnalyzerTest, + ConstantJumpiKeepsMatchingEntryDepthSuccessorsLiftable) { + const std::vector Bytecode = { + 0x60, 0x01, // PUSH1 0x01 + 0x60, 0x06, // PUSH1 0x06 + 0x57, // JUMPI + 0x00, // STOP + 0x5b, // JUMPDEST + 0x00 // STOP + }; + + const EVMAnalyzer Analyzer = analyzeBytecode(Bytecode); + + const auto *EntryBlock = findBlock(Analyzer, 0); + const auto *FallthroughBlock = findBlock(Analyzer, 5); + const auto *JumpDestBlock = findBlock(Analyzer, 6); + ASSERT_NE(EntryBlock, nullptr); + ASSERT_NE(FallthroughBlock, nullptr); + ASSERT_NE(JumpDestBlock, nullptr); + + EXPECT_TRUE(EntryBlock->HasConditionalJump); + EXPECT_TRUE(EntryBlock->HasConstantJump); + EXPECT_FALSE(EntryBlock->HasDynamicJump); + expectPCList(EntryBlock->Successors, {5, 6}); + + EXPECT_EQ(FallthroughBlock->ResolvedEntryStackDepth, 0); + EXPECT_TRUE(FallthroughBlock->CanLiftStack); + expectPCList(FallthroughBlock->Predecessors, {0}); + + EXPECT_TRUE(JumpDestBlock->IsJumpDest); + EXPECT_EQ(JumpDestBlock->ResolvedEntryStackDepth, 0); + EXPECT_TRUE(JumpDestBlock->CanLiftStack); + expectPCList(JumpDestBlock->Predecessors, {0}); +} + +TEST(EVMJITFrontendAnalyzerTest, + DynamicJumpForcesReachableJumpDestsToFallback) { + const std::vector Bytecode = { + 0x60, 0x01, // PUSH1 0x01 + 0x60, 0x07, // PUSH1 0x07 + 0x57, // JUMPI + 0x35, // CALLDATALOAD + 0x56, // JUMP + 0x5b, // JUMPDEST + 0x00 // STOP + }; + + const EVMAnalyzer Analyzer = analyzeBytecode(Bytecode); + + const auto *EntryBlock = findBlock(Analyzer, 0); + const auto *DynamicJumpBlock = findBlock(Analyzer, 5); + const auto *JumpDestBlock = findBlock(Analyzer, 7); + ASSERT_NE(EntryBlock, nullptr); + ASSERT_NE(DynamicJumpBlock, nullptr); + ASSERT_NE(JumpDestBlock, nullptr); + + EXPECT_TRUE(Analyzer.hasUnknownDynamicJumpTargets()); + EXPECT_TRUE(EntryBlock->HasConditionalJump); + EXPECT_TRUE(DynamicJumpBlock->HasDynamicJump); + EXPECT_TRUE(DynamicJumpBlock->CanLiftStack); + + EXPECT_TRUE(JumpDestBlock->IsJumpDest); + EXPECT_EQ(JumpDestBlock->ResolvedEntryStackDepth, 0); + EXPECT_FALSE(JumpDestBlock->CanLiftStack); +} + +TEST(EVMJITFrontendAnalyzerTest, HiddenEntryPrefixKeepsStaticMergesLiftable) { + const std::vector Bytecode = { + 0x60, 0xaa, // PUSH1 preserved prefix + 0x60, 0x01, // PUSH1 cond + 0x60, 0x0a, // PUSH1 jumpdest + 0x57, // JUMPI + 0x60, 0x00, // PUSH1 0x00 + 0x00, // STOP + 0x5b, // JUMPDEST + 0x00 // STOP + }; + + const EVMAnalyzer Analyzer = analyzeBytecode(Bytecode); + const auto *FallthroughBlock = findBlock(Analyzer, 7); + const auto *JumpDestBlock = findBlock(Analyzer, 10); + ASSERT_NE(FallthroughBlock, nullptr); + ASSERT_NE(JumpDestBlock, nullptr); + + EXPECT_EQ(FallthroughBlock->ResolvedEntryStackDepth, 1); + EXPECT_EQ(FallthroughBlock->EntryStackDepth, 0); + EXPECT_TRUE(FallthroughBlock->CanLiftStack); + + EXPECT_EQ(JumpDestBlock->ResolvedEntryStackDepth, 1); + EXPECT_EQ(JumpDestBlock->EntryStackDepth, 0); + EXPECT_TRUE(JumpDestBlock->CanLiftStack); +} + +TEST(EVMJITFrontendAnalyzerTest, MergeDepthConflictDisablesLiftedEntry) { + const std::vector Bytecode = { + 0x60, 0x01, // PUSH1 0x01 + 0x60, 0x0a, // PUSH1 0x0a + 0x57, // JUMPI + 0x60, 0x02, // PUSH1 0x02 + 0x60, 0x0a, // PUSH1 0x0a + 0x56, // JUMP + 0x5b, // JUMPDEST + 0x00 // STOP + }; + + const EVMAnalyzer Analyzer = analyzeBytecode(Bytecode); + + const auto *EntryBlock = findBlock(Analyzer, 0); + const auto *FallthroughBlock = findBlock(Analyzer, 5); + const auto *MergeBlock = findBlock(Analyzer, 10); + ASSERT_NE(EntryBlock, nullptr); + ASSERT_NE(FallthroughBlock, nullptr); + ASSERT_NE(MergeBlock, nullptr); + + EXPECT_TRUE(EntryBlock->HasConditionalJump); + EXPECT_EQ(FallthroughBlock->ResolvedEntryStackDepth, 0); + EXPECT_EQ(FallthroughBlock->ResolvedExitStackDepth, 1); + EXPECT_TRUE(FallthroughBlock->CanLiftStack); + + EXPECT_TRUE(MergeBlock->IsJumpDest); + EXPECT_EQ(MergeBlock->ResolvedEntryStackDepth, -1); + EXPECT_EQ(MergeBlock->ResolvedExitStackDepth, -1); + EXPECT_TRUE(MergeBlock->HasInconsistentEntryDepth); + EXPECT_FALSE(MergeBlock->CanLiftStack); + expectPCList(MergeBlock->Predecessors, {0, 5}); +} + +TEST(EVMJITFrontendAnalyzerTest, + InconsistentMergeInvalidatesReachableSuccessors) { + const std::vector Bytecode = { + 0x60, 0x01, // PUSH1 0x01 + 0x60, 0x0a, // PUSH1 0x0a + 0x57, // JUMPI + 0x60, 0x02, // PUSH1 0x02 + 0x60, 0x0a, // PUSH1 0x0a + 0x56, // JUMP + 0x5b, // JUMPDEST + 0x60, 0x03, // PUSH1 0x03 + 0x5b, // JUMPDEST + 0x00 // STOP + }; + + const EVMAnalyzer Analyzer = analyzeBytecode(Bytecode); + + const auto *MergeBlock = findBlock(Analyzer, 10); + const auto *SuccessorBlock = findBlock(Analyzer, 13); + ASSERT_NE(MergeBlock, nullptr); + ASSERT_NE(SuccessorBlock, nullptr); + + EXPECT_EQ(MergeBlock->ResolvedEntryStackDepth, -1); + EXPECT_EQ(MergeBlock->ResolvedExitStackDepth, -1); + EXPECT_TRUE(MergeBlock->HasInconsistentEntryDepth); + EXPECT_FALSE(MergeBlock->CanLiftStack); + + EXPECT_TRUE(SuccessorBlock->IsJumpDest); + EXPECT_EQ(SuccessorBlock->ResolvedEntryStackDepth, -1); + EXPECT_EQ(SuccessorBlock->ResolvedExitStackDepth, -1); + EXPECT_TRUE(SuccessorBlock->HasInconsistentEntryDepth); + EXPECT_FALSE(SuccessorBlock->CanLiftStack); + expectPCList(SuccessorBlock->Predecessors, {10}); +} + +TEST(EVMJITFrontendVisitorTest, + MaterializedBlockKeepsPopDupSwapAndAddOnLogicalStack) { + const std::vector Bytecode = { + 0x60, 0xaa, // PUSH1 0xaa + 0x60, 0xbb, // PUSH1 0xbb + 0x60, 0x01, // PUSH1 cond + 0x60, 0x0e, // PUSH1 jumpdest + 0x57, // JUMPI + 0x60, 0xcc, // PUSH1 0xcc + 0x60, 0x0e, // PUSH1 jumpdest + 0x56, // JUMP + 0x5b, // JUMPDEST + 0x50, // POP + 0x80, // DUP1 + 0x90, // SWAP1 + 0x01, // ADD + 0x00 // STOP + }; + + const EVMAnalyzer Analyzer = analyzeBytecode(Bytecode); + const auto *TargetBlock = findBlock(Analyzer, 14); + ASSERT_NE(TargetBlock, nullptr); + EXPECT_FALSE(TargetBlock->CanLiftStack); + EXPECT_EQ(TargetBlock->ResolvedEntryStackDepth, -1); + EXPECT_EQ(TargetBlock->ResolvedExitStackDepth, -1); + EXPECT_TRUE(TargetBlock->HasInconsistentEntryDepth); + + COMPILER::EVMFrontendContext Ctx; + Ctx.setRevision(EVMC_CANCUN); + Ctx.setBytecode(reinterpret_cast(Bytecode.data()), + Bytecode.size()); + + MockEVMBuilder Builder; + COMPILER::EVMByteCodeVisitor Visitor(Builder, &Ctx); + EXPECT_TRUE(Visitor.compile()); + EXPECT_FALSE(Builder.Trapped); + EXPECT_FALSE(Builder.Undefined); + + const auto &PopStats = Builder.accessStats(OP_POP); + EXPECT_EQ(PopStats.StackPopCount, 0U); + EXPECT_EQ(PopStats.StackGetCount, 0U); + EXPECT_EQ(PopStats.StackSetCount, 0U); + + const auto &DupStats = Builder.accessStats(OP_DUP1); + EXPECT_EQ(DupStats.StackPopCount, 0U); + EXPECT_EQ(DupStats.StackGetCount, 0U); + EXPECT_EQ(DupStats.StackSetCount, 0U); + + const auto &SwapStats = Builder.accessStats(OP_SWAP1); + EXPECT_EQ(SwapStats.StackPopCount, 0U); + EXPECT_EQ(SwapStats.StackGetCount, 0U); + EXPECT_EQ(SwapStats.StackSetCount, 0U); + + const auto &AddStats = Builder.accessStats(OP_ADD); + EXPECT_EQ(AddStats.StackPopCount, 0U); + EXPECT_EQ(AddStats.StackGetCount, 0U); + EXPECT_EQ(AddStats.StackSetCount, 0U); +} + +TEST(EVMJITFrontendVisitorTest, + LiftedJumpDestEntryDoesNotDependOnRuntimeStackDepth) { + const std::vector Bytecode = { + 0x60, 0x01, // PUSH1 0x01 + 0x5b, // JUMPDEST + 0x80, // DUP1 + 0x50, // POP + 0x00 // STOP + }; + + const EVMAnalyzer Analyzer = analyzeBytecode(Bytecode); + const auto *EntryBlock = findBlock(Analyzer, 0); + const auto *JumpDestBlock = findBlock(Analyzer, 2); + ASSERT_NE(EntryBlock, nullptr); + ASSERT_NE(JumpDestBlock, nullptr); + EXPECT_TRUE(EntryBlock->CanLiftStack); + EXPECT_TRUE(JumpDestBlock->CanLiftStack); + EXPECT_EQ(JumpDestBlock->ResolvedEntryStackDepth, 1); + + COMPILER::EVMFrontendContext Ctx; + Ctx.setRevision(EVMC_CANCUN); + Ctx.setBytecode(reinterpret_cast(Bytecode.data()), + Bytecode.size()); + + MockEVMBuilder Builder; + Builder.enableRuntimeStackChecks(); + COMPILER::EVMByteCodeVisitor Visitor(Builder, &Ctx); + EXPECT_TRUE(Visitor.compile()); + EXPECT_FALSE(Builder.Trapped); + EXPECT_FALSE(Builder.Undefined); +} + +TEST(EVMJITFrontendVisitorTest, + ImplicitStopMaterializesLiftedStackOnFallthrough) { + const std::vector Bytecode = { + 0x60, 0xaa // PUSH1 0xaa + }; + + const EVMAnalyzer Analyzer = analyzeBytecode(Bytecode); + const auto *EntryBlock = findBlock(Analyzer, 0); + ASSERT_NE(EntryBlock, nullptr); + EXPECT_TRUE(EntryBlock->CanLiftStack); + EXPECT_EQ(EntryBlock->ResolvedExitStackDepth, 1); + + COMPILER::EVMFrontendContext Ctx; + Ctx.setRevision(EVMC_CANCUN); + Ctx.setBytecode(reinterpret_cast(Bytecode.data()), + Bytecode.size()); + + MockEVMBuilder Builder; + COMPILER::EVMByteCodeVisitor Visitor(Builder, &Ctx); + EXPECT_TRUE(Visitor.compile()); + EXPECT_FALSE(Builder.Trapped); + EXPECT_FALSE(Builder.Undefined); + EXPECT_EQ(Builder.runtimeStackDepth(), 1U); + EXPECT_EQ(Builder.topStackValue()[0], 0xaaU); +} + +TEST(EVMJITFrontendVisitorTest, + UndefinedInstructionAfterProducerDoesNotTriggerStackOverflowTrap) { + const std::vector Bytecode = { + 0x30, // ADDRESS + 0x2a // undefined + }; + + const EVMAnalyzer Analyzer = analyzeBytecode(Bytecode); + const auto *EntryBlock = findBlock(Analyzer, 0); + ASSERT_NE(EntryBlock, nullptr); + EXPECT_TRUE(EntryBlock->HasUndefinedInstr); + EXPECT_EQ(EntryBlock->MaxStackHeight, 1); + + COMPILER::EVMFrontendContext Ctx; + Ctx.setRevision(EVMC_CANCUN); + Ctx.setBytecode(reinterpret_cast(Bytecode.data()), + Bytecode.size()); + + MockEVMBuilder Builder; + Builder.enableRuntimeStackChecks(); + COMPILER::EVMByteCodeVisitor Visitor(Builder, &Ctx); + EXPECT_TRUE(Visitor.compile()); + EXPECT_FALSE(Builder.Trapped); + EXPECT_TRUE(Builder.Undefined); +} + +TEST(EVMJITFrontendVisitorTest, TruncatedPushIsRightPaddedWithZeros) { + const std::vector Bytecode = { + 0x62, 0x12, 0x34 // PUSH3 0x12 0x34 + }; + + const EVMAnalyzer Analyzer = analyzeBytecode(Bytecode); + const auto *EntryBlock = findBlock(Analyzer, 0); + ASSERT_NE(EntryBlock, nullptr); + EXPECT_EQ(EntryBlock->ResolvedExitStackDepth, 1); + + COMPILER::EVMFrontendContext Ctx; + Ctx.setRevision(EVMC_CANCUN); + Ctx.setBytecode(reinterpret_cast(Bytecode.data()), + Bytecode.size()); + + MockEVMBuilder Builder; + COMPILER::EVMByteCodeVisitor Visitor(Builder, &Ctx); + EXPECT_TRUE(Visitor.compile()); + EXPECT_FALSE(Builder.Trapped); + EXPECT_FALSE(Builder.Undefined); + ASSERT_TRUE(Builder.hasLastPushValue()); + EXPECT_EQ(Builder.lastPushValue()[0], 0x123400ULL); + EXPECT_EQ(Builder.lastPushValue()[1], 0ULL); + EXPECT_EQ(Builder.lastPushValue()[2], 0ULL); + EXPECT_EQ(Builder.lastPushValue()[3], 0ULL); +} + +} // namespace diff --git a/src/vm/dt_evmc_vm.cpp b/src/vm/dt_evmc_vm.cpp index 57f6e4d72..0e38436f8 100644 --- a/src/vm/dt_evmc_vm.cpp +++ b/src/vm/dt_evmc_vm.cpp @@ -295,10 +295,33 @@ bool shouldUsePersistentModuleCache(const evmc_message *Msg) { Msg->kind != EVMC_CREATE2; } +bool shouldRetryModuleLoadWithFastRA(const DTVM *VM, const Error &Err) { + return VM->Config.Mode == RunMode::MultipassMode && + !VM->Config.DisableMultipassGreedyRA && + Err.getPhase() == ErrorPhase::Compilation && + Err.getSubphase() == ErrorSubphase::RegAlloc; +} + +zen::common::MayBe +loadEVMModuleWithRegAllocRetry(DTVM *VM, const std::string &ModName, + const uint8_t *Code, size_t CodeSize, + evmc_revision Rev) { + auto ModRet = VM->RT->loadEVMModule(ModName, Code, CodeSize, Rev); + if (ModRet || !shouldRetryModuleLoadWithFastRA(VM, ModRet.getError())) { + return ModRet; + } + + RuntimeConfig RetryConfig = VM->Config; + RetryConfig.DisableMultipassGreedyRA = true; + ScopedConfig Retry(VM->RT.get(), RetryConfig); + return VM->RT->loadEVMModule(ModName, Code, CodeSize, Rev); +} + EVMModule *loadTransientModule(DTVM *VM, const uint8_t *Code, size_t CodeSize, evmc_revision Rev) { std::string ModName = "tmp_mod_" + std::to_string(VM->ModCounter++); - auto ModRet = VM->RT->loadEVMModule(ModName, Code, CodeSize, Rev); + auto ModRet = + loadEVMModuleWithRegAllocRetry(VM, ModName, Code, CodeSize, Rev); if (!ModRet) return nullptr; return *ModRet; @@ -362,7 +385,8 @@ EVMModule *findModuleCached(DTVM *VM, const uint8_t *Code, size_t CodeSize, } std::string ModName = "mod_" + std::to_string(VM->ModCounter++); - auto ModRet = VM->RT->loadEVMModule(ModName, Code, CodeSize, Rev); + auto ModRet = + loadEVMModuleWithRegAllocRetry(VM, ModName, Code, CodeSize, Rev); if (!ModRet) return nullptr; Mod = *ModRet; @@ -527,26 +551,6 @@ evmc_result execute(evmc_vm *EVMInstance, const evmc_host_interface *Host, return evmc_make_result(EVMC_FAILURE, 0, 0, nullptr, 0); } -#ifdef ZEN_ENABLE_JIT_PRECOMPILE_FALLBACK - // Use interpreter mode for bytecode that would be too expensive to JIT. - // The EVMAnalyzer performs a pattern-aware O(n) scan that detects: - // - raw bytecode size / estimated MIR instruction count too large - // - high density of RA-expensive opcodes (SHL/SHR/SAR/MUL/SIGNEXTEND) - // - long consecutive runs of RA-expensive ops - // - DUP-induced feedback loops (b0 pattern) - std::unique_ptr TempConfig; - if (VM->Config.Mode == RunMode::MultipassMode) { - COMPILER::EVMAnalyzer Analyzer(Rev); - Analyzer.analyze(Code, CodeSize); - const auto &JITResult = Analyzer.getJITSuitability(); - if (JITResult.ShouldFallback) { - RuntimeConfig NewConfig = VM->Config; - NewConfig.Mode = RunMode::InterpMode; - TempConfig = std::make_unique(VM->RT.get(), NewConfig); - } - } -#endif // ZEN_ENABLE_JIT_PRECOMPILE_FALLBACK - // Module lookup: L0 -> L1 -> Cold (shared with interpreter path) bool IsTransientMod = false; EVMModule *Mod = @@ -556,6 +560,18 @@ evmc_result execute(evmc_vm *EVMInstance, const evmc_host_interface *Host, } ModuleGuard ModGuard(VM, Mod, IsTransientMod); +#ifdef ZEN_ENABLE_JIT_PRECOMPILE_FALLBACK + // The expensive analyzer-based fallback decision is cached on the module at + // load time and lazily initialized at most once per module. + std::unique_ptr TempConfig; + if (VM->Config.Mode == RunMode::MultipassMode && + Mod->shouldInterpretInMultipass()) { + RuntimeConfig NewConfig = VM->Config; + NewConfig.Mode = RunMode::InterpMode; + TempConfig = std::make_unique(VM->RT.get(), NewConfig); + } +#endif // ZEN_ENABLE_JIT_PRECOMPILE_FALLBACK + // Instance reuse (shared only for cacheable top-level calls) auto *TheInst = getOrCreateInstance(VM, Mod, Rev, Msg->depth); if (!TheInst) {