Skip to content

perf(evm): add constant-shift fast paths #388

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
starwarfan wants to merge 1 commit into
base: main
Choose a base branch
from
+122 −6
Draft

perf(evm): add constant-shift fast paths #388

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 122 additions & 6 deletions src/compiler/evm_frontend/evm_mir_compiler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,14 @@
#include "compiler/evm_frontend/evm_mir_compiler.h"
#include "action/evm_bytecode_visitor.h"
#include "compiler/evm_frontend/evm_imported.h"
#include "compiler/mir/constants.h"
#include "compiler/mir/module.h"
#include "evm/gas_storage_cost.h"
#include "runtime/evm_instance.h"
#include "utils/hash_utils.h"
#include "llvm/Support/Casting.h"
#include <cstring>
#include <optional>

#ifdef ZEN_ENABLE_EVM_GAS_REGISTER
#include "compiler/llvm-prebuild/Target/X86/X86Subtarget.h"
Expand Down Expand Up @@ -1876,6 +1879,18 @@ EVMMirBuilder::handleClz(const Operand &ValueOp) {
RuntimeFunctions.GetClz, ValueOp);
}

namespace {
// Extract constant shift amount from MInstruction if it is a constant.
std::optional<uint64_t> getConstShiftAmount(MInstruction *Inst) {
if (auto *CI = llvm::dyn_cast<ConstantInstruction>(Inst)) {
if (auto *IntConst = llvm::dyn_cast<MConstantInt>(&CI->getConstant())) {
return IntConst->getValue().getZExtValue();
}
}
return std::nullopt;
}
} // namespace

EVMMirBuilder::U256Inst
EVMMirBuilder::handleLeftShift(const U256Inst &Value, MInstruction *ShiftAmount,
MInstruction *IsLargeShift) {
Expand All @@ -1884,6 +1899,41 @@ EVMMirBuilder::handleLeftShift(const U256Inst &Value, MInstruction *ShiftAmount,
U256Inst Result = {};

MInstruction *Zero = createIntConstInstruction(MirI64Type, 0);

// Fast path: constant shift amount — direct limb logic, no Select/cmp loops.
if (auto ShiftOpt = getConstShiftAmount(ShiftAmount)) {
uint64_t Shift = *ShiftOpt;
if (Shift >= 256) {
for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I)
Result[I] = Zero;
return Result;
}
uint64_t CompShift = Shift / 64;
uint64_t ShiftMod = Shift % 64;
uint64_t RemainingBits = (ShiftMod == 0) ? 0 : (64 - ShiftMod);
for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) {
MInstruction *R = Zero;
if (I >= CompShift) {
size_t SrcIdx = I - CompShift;
MInstruction *SrcVal = Value[SrcIdx];
MInstruction *Shifted = createInstruction<BinaryInstruction>(
false, OP_shl, MirI64Type, SrcVal,
createIntConstInstruction(MirI64Type, ShiftMod));
if (SrcIdx > 0 && RemainingBits > 0) {
MInstruction *Carry = createInstruction<BinaryInstruction>(
false, OP_ushr, MirI64Type, Value[SrcIdx - 1],
createIntConstInstruction(MirI64Type, RemainingBits));
R = createInstruction<BinaryInstruction>(false, OP_or, MirI64Type,
Shifted, Carry);
} else {
R = Shifted;
Comment on lines +1914 to +1929
Copy link

Copilot AI Mar 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the constant-SHL fast path, createIntConstInstruction(..., ShiftMod) and createIntConstInstruction(..., RemainingBits) are called inside the loop, creating a new OP_const each iteration even though the value is invariant. Hoist these constant instructions outside the loop (and consider skipping the shift/or entirely when ShiftMod == 0 by directly using the source limb) to keep the fast path from inflating MIR instruction count.

Suggested change
for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) {
MInstruction *R = Zero;
if (I >= CompShift) {
size_t SrcIdx = I - CompShift;
MInstruction *SrcVal = Value[SrcIdx];
MInstruction *Shifted = createInstruction<BinaryInstruction>(
false, OP_shl, MirI64Type, SrcVal,
createIntConstInstruction(MirI64Type, ShiftMod));
if (SrcIdx > 0 && RemainingBits > 0) {
MInstruction *Carry = createInstruction<BinaryInstruction>(
false, OP_ushr, MirI64Type, Value[SrcIdx - 1],
createIntConstInstruction(MirI64Type, RemainingBits));
R = createInstruction<BinaryInstruction>(false, OP_or, MirI64Type,
Shifted, Carry);
} else {
R = Shifted;
// Hoist loop-invariant constant instructions out of the limb loop.
MInstruction *ShiftModConst = nullptr;
MInstruction *RemainingBitsConst = nullptr;
if (ShiftMod != 0) {
ShiftModConst = createIntConstInstruction(MirI64Type, ShiftMod);
}
if (RemainingBits > 0) {
RemainingBitsConst =
createIntConstInstruction(MirI64Type, RemainingBits);
}
for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) {
MInstruction *R = Zero;
if (I >= CompShift) {
size_t SrcIdx = I - CompShift;
if (ShiftMod == 0) {
// Pure limb shift (multiple of 64): no intra-limb shift/carry needed.
R = Value[SrcIdx];
} else {
MInstruction *SrcVal = Value[SrcIdx];
MInstruction *Shifted = createInstruction<BinaryInstruction>(
false, OP_shl, MirI64Type, SrcVal, ShiftModConst);
if (SrcIdx > 0 && RemainingBitsConst) {
MInstruction *Carry = createInstruction<BinaryInstruction>(
false, OP_ushr, MirI64Type, Value[SrcIdx - 1],
RemainingBitsConst);
R = createInstruction<BinaryInstruction>(false, OP_or, MirI64Type,
Shifted, Carry);
} else {
R = Shifted;
}

Copilot uses AI. Check for mistakes.
}
}
Result[I] = protectUnsafeValue(R, MirI64Type);
}
return Result;
}
Comment on lines +1903 to +1935
Copy link

Copilot AI Mar 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The constant-shift fast path derives Shift only from ShiftAmount (low 64 bits) and bypasses the IsLargeShift guard. This breaks EVM semantics when the 256-bit shift has any high limb set but Shift[0] < 256 (e.g. shift = 2^64 => ShiftAmount constant 0 while IsLargeShift should force the result to 0). Ensure the fast path still applies IsLargeShift (e.g., select between 0 and the computed result per limb) or move constant-shift handling up to handleShift() where all 4 limbs are available.

Suggested change
// Fast path: constant shift amount — direct limb logic, no Select/cmp loops.
if (auto ShiftOpt = getConstShiftAmount(ShiftAmount)) {
uint64_t Shift = *ShiftOpt;
if (Shift >= 256) {
for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I)
Result[I] = Zero;
return Result;
}
uint64_t CompShift = Shift / 64;
uint64_t ShiftMod = Shift % 64;
uint64_t RemainingBits = (ShiftMod == 0) ? 0 : (64 - ShiftMod);
for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) {
MInstruction *R = Zero;
if (I >= CompShift) {
size_t SrcIdx = I - CompShift;
MInstruction *SrcVal = Value[SrcIdx];
MInstruction *Shifted = createInstruction<BinaryInstruction>(
false, OP_shl, MirI64Type, SrcVal,
createIntConstInstruction(MirI64Type, ShiftMod));
if (SrcIdx > 0 && RemainingBits > 0) {
MInstruction *Carry = createInstruction<BinaryInstruction>(
false, OP_ushr, MirI64Type, Value[SrcIdx - 1],
createIntConstInstruction(MirI64Type, RemainingBits));
R = createInstruction<BinaryInstruction>(false, OP_or, MirI64Type,
Shifted, Carry);
} else {
R = Shifted;
}
}
Result[I] = protectUnsafeValue(R, MirI64Type);
}
return Result;
}
// Note: We deliberately avoid a constant-shift fast path here because
// deriving the 256-bit shift solely from the low 64-bit ShiftAmount
// can bypass the IsLargeShift guard and break EVM semantics when any
// high limb of the shift value is non-zero. All shifts are handled
// by the generic implementation below, which correctly applies
// IsLargeShift to enforce zeroing for large shifts.

Copilot uses AI. Check for mistakes.

MInstruction *One = createIntConstInstruction(MirI64Type, 1);
MInstruction *Const64 = createIntConstInstruction(MirI64Type, 64);

Expand Down Expand Up @@ -2019,6 +2069,41 @@ EVMMirBuilder::handleLogicalRightShift(const U256Inst &Value,
U256Inst Result = {};

MInstruction *Zero = createIntConstInstruction(MirI64Type, 0);

// Fast path: constant shift amount — direct limb logic, no Select/cmp loops.
if (auto ShiftOpt = getConstShiftAmount(ShiftAmount)) {
uint64_t Shift = *ShiftOpt;
if (Shift >= 256) {
for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I)
Result[I] = Zero;
return Result;
}
uint64_t CompShift = Shift / 64;
uint64_t ShiftMod = Shift % 64;
uint64_t CarryShift = (ShiftMod == 0) ? 0 : (64 - ShiftMod);
for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) {
Comment on lines +2073 to +2084
Copy link

Copilot AI Mar 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same issue as SHL fast path: the constant-shift path uses only ShiftAmount (low 64 bits) and ignores IsLargeShift, so shifts with any non-zero high limb but small low limb will incorrectly behave like a small shift instead of producing 0 per EVM spec. The fast path should still incorporate IsLargeShift (e.g., per-limb select to 0) to preserve correctness for large 256-bit shift values.

Copilot uses AI. Check for mistakes.
MInstruction *R = Zero;
if (I + CompShift < EVM_ELEMENTS_COUNT) {
size_t SrcIdx = I + CompShift;
MInstruction *SrcVal = Value[SrcIdx];
MInstruction *Shifted = createInstruction<BinaryInstruction>(
false, OP_ushr, MirI64Type, SrcVal,
createIntConstInstruction(MirI64Type, ShiftMod));
if (SrcIdx + 1 < EVM_ELEMENTS_COUNT && CarryShift > 0) {
MInstruction *Carry = createInstruction<BinaryInstruction>(
false, OP_shl, MirI64Type, Value[SrcIdx + 1],
createIntConstInstruction(MirI64Type, CarryShift));
Comment on lines +2084 to +2095
Copy link

Copilot AI Mar 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the constant-SHR fast path, the shift amounts are materialized with createIntConstInstruction inside the per-limb loop, which produces duplicate OP_const instructions for ShiftMod/CarryShift. Hoist these constants outside the loop (and when ShiftMod == 0, avoid generating the shift/or at all by directly using the source limb) to maximize the intended perf win and reduce MIR bloat.

Suggested change
for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) {
MInstruction *R = Zero;
if (I + CompShift < EVM_ELEMENTS_COUNT) {
size_t SrcIdx = I + CompShift;
MInstruction *SrcVal = Value[SrcIdx];
MInstruction *Shifted = createInstruction<BinaryInstruction>(
false, OP_ushr, MirI64Type, SrcVal,
createIntConstInstruction(MirI64Type, ShiftMod));
if (SrcIdx + 1 < EVM_ELEMENTS_COUNT && CarryShift > 0) {
MInstruction *Carry = createInstruction<BinaryInstruction>(
false, OP_shl, MirI64Type, Value[SrcIdx + 1],
createIntConstInstruction(MirI64Type, CarryShift));
// If the shift is a multiple of 64, we only need to move whole limbs.
if (ShiftMod == 0) {
for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) {
MInstruction *R = Zero;
if (I + CompShift < EVM_ELEMENTS_COUNT) {
size_t SrcIdx = I + CompShift;
R = Value[SrcIdx];
}
Result[I] = protectUnsafeValue(R, MirI64Type);
}
return Result;
}
// Hoist loop-invariant shift constants out of the limb loop.
MInstruction *ShiftModConst =
createIntConstInstruction(MirI64Type, ShiftMod);
MInstruction *CarryShiftConst = nullptr;
if (CarryShift > 0)
CarryShiftConst = createIntConstInstruction(MirI64Type, CarryShift);
for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) {
MInstruction *R = Zero;
if (I + CompShift < EVM_ELEMENTS_COUNT) {
size_t SrcIdx = I + CompShift;
MInstruction *SrcVal = Value[SrcIdx];
MInstruction *Shifted = createInstruction<BinaryInstruction>(
false, OP_ushr, MirI64Type, SrcVal, ShiftModConst);
if (SrcIdx + 1 < EVM_ELEMENTS_COUNT && CarryShiftConst != nullptr) {
MInstruction *Carry = createInstruction<BinaryInstruction>(
false, OP_shl, MirI64Type, Value[SrcIdx + 1], CarryShiftConst);

Copilot uses AI. Check for mistakes.
R = createInstruction<BinaryInstruction>(false, OP_or, MirI64Type,
Shifted, Carry);
} else {
R = Shifted;
}
}
Result[I] = protectUnsafeValue(R, MirI64Type);
}
return Result;
}

MInstruction *One = createIntConstInstruction(MirI64Type, 1);
MInstruction *Const64 = createIntConstInstruction(MirI64Type, 64);

Expand Down Expand Up @@ -2146,25 +2231,56 @@ EVMMirBuilder::handleArithmeticRightShift(const U256Inst &Value,
EVMFrontendContext::getMIRTypeFromEVMType(EVMType::UINT64);
U256Inst Result = {};

// Arithmetic right shift: sign-extend when shift >= 256
MInstruction *Zero = createIntConstInstruction(MirI64Type, 0);
MInstruction *AllOnes = createIntConstInstruction(MirI64Type, ~0ULL);

// Check sign bit (bit 63 of highest component)
// Check sign bit (bit 63 of highest component) for large-shift result
MInstruction *HighComponent = Value[EVM_ELEMENTS_COUNT - 1];
MInstruction *Const63 = createIntConstInstruction(MirI64Type, 63);
MInstruction *SignBit = createInstruction<BinaryInstruction>(
false, OP_ushr, MirI64Type, HighComponent, Const63);

// Sign bit is 1 if negative
MInstruction *One = createIntConstInstruction(MirI64Type, 1);
MInstruction *IsNegative = createInstruction<CmpInstruction>(
false, CmpInstruction::Predicate::ICMP_EQ, &Ctx.I64Type, SignBit, One);

// Large shift result: all 1s if negative, all 0s if positive
MInstruction *LargeShiftResult = createInstruction<SelectInstruction>(
false, MirI64Type, IsNegative, AllOnes, Zero);

// Fast path: constant shift amount — direct limb logic, no Select/cmp loops.
if (auto ShiftOpt = getConstShiftAmount(ShiftAmount)) {
uint64_t Shift = *ShiftOpt;
if (Shift >= 256) {
for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I)
Result[I] = LargeShiftResult;
return Result;
}
uint64_t CompShift = Shift / 64;
uint64_t ShiftMod = Shift % 64;
uint64_t CarryShift = (ShiftMod == 0) ? 0 : (64 - ShiftMod);
for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) {
Comment on lines +2248 to +2259
Copy link

Copilot AI Mar 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The SAR constant-shift fast path ignores IsLargeShift and only checks Shift >= 256 based on the low 64 bits. This is incorrect for 256-bit shift values with high limbs set but Shift[0] < 256 (EVM requires full sign-extension result when shift >= 256). Make the fast path depend on IsLargeShift (e.g., select LargeShiftResult vs computed limb result) or lift constant-shift evaluation to handleShift() where the full 256-bit shift is known.

Copilot uses AI. Check for mistakes.
MInstruction *R = LargeShiftResult;
if (I + CompShift < EVM_ELEMENTS_COUNT) {
size_t SrcIdx = I + CompShift;
MInstruction *SrcVal = Value[SrcIdx];
// Use arithmetic shift for the high component (contains sign bit)
bool UseArithShift = (SrcIdx == EVM_ELEMENTS_COUNT - 1);
MInstruction *Shifted = createInstruction<BinaryInstruction>(
false, UseArithShift ? OP_sshr : OP_ushr, MirI64Type, SrcVal,
createIntConstInstruction(MirI64Type, ShiftMod));
if (SrcIdx + 1 < EVM_ELEMENTS_COUNT && CarryShift > 0) {
MInstruction *Carry = createInstruction<BinaryInstruction>(
false, OP_shl, MirI64Type, Value[SrcIdx + 1],
createIntConstInstruction(MirI64Type, CarryShift));
R = createInstruction<BinaryInstruction>(false, OP_or, MirI64Type,
Shifted, Carry);
Comment on lines +2264 to +2274
Copy link

Copilot AI Mar 10, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the constant-SAR fast path, createIntConstInstruction(..., ShiftMod) / createIntConstInstruction(..., CarryShift) are created inside the loop even though they are loop-invariant, adding extra OP_const nodes. Hoist the constants outside the loop (and if ShiftMod == 0, avoid emitting a redundant shift-by-0 instruction) so this path stays minimal.

Copilot uses AI. Check for mistakes.
} else {
R = Shifted;
}
}
Result[I] = protectUnsafeValue(R, MirI64Type);
}
return Result;
}

// intra-component shifts = shift % 64
// shift_comp = shift / 64 (which component index shift from)
MInstruction *Const64 = createIntConstInstruction(MirI64Type, 64);
Expand Down
Loading