chore: merge shadow branch fixes and improvements to main

.github/workflows/auto-release.yml

@@ -184,9 +184,9 @@ jobs:
     - name: Build zeam natively
       run: |
         if [ "${{ matrix.arch }}" = "amd64" ]; then
-          zig build -Doptimize=ReleaseFast -Dtarget=x86_64-linux-gnu -Dcpu=baseline -Dgit_version="$(git rev-parse --short HEAD)"
+          zig build -Doptimize=ReleaseSafe -Dtarget=x86_64-linux-gnu -Dcpu=baseline -Dgit_version="$(git rev-parse --short HEAD)"
         else
-          zig build -Doptimize=ReleaseFast -Dtarget=aarch64-linux-gnu -Dcpu=baseline -Dgit_version="$(git rev-parse --short HEAD)"
+          zig build -Doptimize=ReleaseSafe -Dtarget=aarch64-linux-gnu -Dcpu=baseline -Dgit_version="$(git rev-parse --short HEAD)"
         fi
 
     - name: Build and push Docker image with pre-built binary (${{ matrix.arch }})

.github/workflows/ci.yml

@@ -18,6 +18,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: recursive
 
     - name: Set up Zig
       uses: mlugg/[email protected]
@@ -80,6 +82,8 @@ jobs:
         os: [ubuntu-latest, macos-latest]
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: recursive
 
     - name: Free disk space (Ubuntu)
       if: runner.os == 'Linux'
@@ -149,6 +153,8 @@ jobs:
         os: [ubuntu-latest, macos-latest]
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: recursive
 
     - name: Free disk space (Ubuntu)
       if: runner.os == 'Linux'
@@ -318,6 +324,8 @@ jobs:
         os: [ubuntu-latest, macos-latest]
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: recursive
 
     - name: Free disk space (Ubuntu)
       if: runner.os == 'Linux'
@@ -388,6 +396,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
       with:
+        submodules: recursive
         fetch-depth: 0  # Fetch full history to get git commit info
 
     - name: Free disk space (Ubuntu)
@@ -439,7 +448,7 @@ jobs:
       run: |
         max_attempts=3
         for attempt in $(seq 1 $max_attempts); do
-          if zig build -Doptimize=ReleaseFast -Dgit_version="$(git rev-parse --short HEAD)"; then
+          if zig build -Doptimize=ReleaseSafe -Dgit_version="$(git rev-parse --short HEAD)"; then
             echo "Successfully built on attempt $attempt"
             exit 0
           fi

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "leanSpec"]
 	path = leanSpec
 	url = https://github.com/leanEthereum/leanSpec
+[submodule "test-keys"]
+	path = test-keys
+	url = https://github.com/blockblaz/zeam-test-keys.git

Dockerfile

@@ -93,7 +93,7 @@ RUN --mount=type=cache,target=/root/.cache/zig \
     else \
         GIT_VERSION=$(echo "$GIT_VERSION" | head -c 7); \
     fi && \
-    zig build -Doptimize=ReleaseFast -Dgit_version="$GIT_VERSION"
+    zig build -Doptimize=ReleaseSafe -Dgit_version="$GIT_VERSION"
 
 # Intermediate stage to prepare runtime libraries
 FROM ubuntu:24.04 AS runtime-prep

README.md

@@ -109,7 +109,7 @@ Zeam is developing and contributing to the Zig Ethereum ecosystem. These librari
 
 | Library | Description |
 |---------|-------------|
-| [ssz.zig](https://github.com/blockblaz/ssz.zig) | SSZ serialization with configurable hash function |
+| [ssz.zig](https://github.com/blockblaz/ssz.zig) | SSZ serialization with configurable hash function (SHA256 or Poseidon2) |
 | [zig-snappy](https://github.com/blockblaz/zig-snappy) / [snappyframesz](https://github.com/blockblaz/snappyframesz) | Snappy compression |
 | [zig-libp2p-pocs](https://github.com/blockblaz/zig-libp2p-pocs) | Zig ↔ Rust libp2p interop |
 | [hash-sigz](https://github.com/blockblaz/hash-sigz) | Hash-based signature schemes |
@@ -143,6 +143,8 @@ To include the git version in the binary:
 zig build -Doptimize=ReleaseFast -Dgit_version="$(git rev-parse --short HEAD)"
 ```
 
+
+> To use Poseidon2 as the SSZ hash function instead of SHA256, see [resources/poseidon.md](resources/poseidon.md).
 ### Running the Prover Demo
 
 ```bash
@@ -172,6 +174,24 @@ docker build -f Dockerfile.prebuilt \
   -t blockblaz/zeam:latest .
 ```
 
+### Troubleshooting
+
+**Build fails with `EndOfStream` error**
+
+If you encounter errors like:
+```
+error: invalid HTTP response: EndOfStream
+```
+
+This may be caused by proxy environment variables interfering with Zig's HTTP client (related to the [Zig HTTP connection pool bug](https://github.com/ziglang/zig/issues/21316) mentioned above).
+
+Try building without proxy settings:
+```bash
+env -u https_proxy -u HTTPS_PROXY -u http_proxy -u HTTP_PROXY \
+    -u all_proxy -u ALL_PROXY -u no_proxy -u NO_PROXY \
+    zig build -Doptimize=ReleaseFast
+```
+
 ---
 
 ## Running a Local Devnet

build.zig

@@ -86,6 +86,8 @@ pub fn build(b: *Builder) !void {
     // LTO option (disabled by default for faster builds)
     const enable_lto = b.option(bool, "lto", "Enable Link Time Optimization (slower builds, smaller binaries)") orelse false;
 
+    // xev backend override is no longer needed — Dynamic API handles runtime detection.
+
     // add ssz
     const ssz = b.dependency("ssz", .{
         .target = target,
@@ -156,6 +158,8 @@ pub fn build(b: *Builder) !void {
     build_options.addOption(bool, "has_openvm", prover == .openvm or prover == .all);
     const use_poseidon = b.option(bool, "use_poseidon", "Use Poseidon SSZ hasher (default: false)") orelse false;
     build_options.addOption(bool, "use_poseidon", use_poseidon);
+    // Absolute path to test-keys for pre-generated validator keys
+    build_options.addOption([]const u8, "test_keys_path", b.pathFromRoot("test-keys/hash-sig-keys"));
     const build_options_module = build_options.createModule();
 
     // add zeam-utils
@@ -240,6 +244,7 @@ pub fn build(b: *Builder) !void {
         .target = target,
         .optimize = optimize,
     });
+    zeam_key_manager.addImport("build_options", build_options_module);
     zeam_key_manager.addImport("@zeam/xmss", zeam_xmss);
     zeam_key_manager.addImport("@zeam/types", zeam_types);
     zeam_key_manager.addImport("@zeam/utils", zeam_utils);
@@ -421,6 +426,10 @@ pub fn build(b: *Builder) !void {
     tools_cli_exe.root_module.addImport("enr", enr);
     tools_cli_exe.root_module.addImport("build_options", build_options_module);
     tools_cli_exe.root_module.addImport("simargs", simargs);
+    tools_cli_exe.root_module.addImport("@zeam/xmss", zeam_xmss);
+    tools_cli_exe.root_module.addImport("@zeam/types", zeam_types);
+    tools_cli_exe.step.dependOn(&build_rust_lib_steps.step);
+    addRustGlueLib(b, tools_cli_exe, target, prover);
 
     const install_tools_cli = b.addInstallArtifact(tools_cli_exe, .{});
     tools_step.dependOn(&install_tools_cli.step);
@@ -602,6 +611,10 @@ pub fn build(b: *Builder) !void {
         .root_module = tools_cli_exe.root_module,
     });
     tools_cli_tests.root_module.addImport("enr", enr);
+    tools_cli_tests.root_module.addImport("@zeam/xmss", zeam_xmss);
+    tools_cli_tests.root_module.addImport("@zeam/types", zeam_types);
+    tools_cli_tests.step.dependOn(&build_rust_lib_steps.step);
+    addRustGlueLib(b, tools_cli_tests, target, prover);
     const run_tools_cli_test = b.addRunArtifact(tools_cli_tests);
     setTestRunLabelFromCompile(b, run_tools_cli_test, tools_cli_tests);
     tools_test_step.dependOn(&run_tools_cli_test.step);
@@ -722,6 +735,8 @@ fn build_zkvm_targets(
     build_options_module: *std.Build.Module,
     use_poseidon: bool,
 ) !void {
+    // zkvm targets (riscv32-freestanding-none) require ReleaseFast; ReleaseSafe
+    // triggers "invalid operand for inline asm constraint 'i'" in LLVM on riscv32.
     const optimize = .ReleaseFast;
 
     for (zkvm_targets) |zkvm_target| {
@@ -839,7 +854,7 @@ fn build_zkvm_targets(
                 .root_module = b.createModule(.{
                     .root_source_file = b.path("build/risc0.zig"),
                     .target = host_target,
-                    .optimize = .ReleaseSafe,
+                    .optimize = .ReleaseFast,
                 }),
             });
             const run_risc0_postbuild_gen_step = b.addRunArtifact(risc0_postbuild_gen);

build.zig.zon

@@ -2,6 +2,7 @@
     .name = .zeam,
     .fingerprint = 0x243fd12cc5f554a5,
     .version = "0.0.0",
+    .minimum_zig_version = "0.15.0",
     .dependencies = .{
         .ssz = .{
             .url = "git+https://github.com/blockblaz/ssz.zig#c5394395dd7d0f8eda685c4723ad25ebbf550570",

pkgs/api/src/event_broadcaster.zig

@@ -240,6 +240,7 @@ test "event broadcaster basic functionality" {
     // Test broadcasting an event (writes to the socket; read end fds[0] is left open so the write succeeds)
     const proto_block = types.ProtoBlock{
         .slot = 123,
+        .proposer_index = 0,
         .blockRoot = [_]u8{1} ** 32,
         .parentRoot = [_]u8{2} ** 32,
         .stateRoot = [_]u8{3} ** 32,

pkgs/api/src/events.zig

@@ -205,6 +205,7 @@ test "serialize new head event" {
 
     const proto_block = types.ProtoBlock{
         .slot = 123,
+        .proposer_index = 0,
         .blockRoot = [_]u8{1} ** 32,
         .parentRoot = [_]u8{2} ** 32,
         .stateRoot = [_]u8{3} ** 32,
@@ -271,6 +272,7 @@ fn makeSampleChainEvent(allocator: Allocator, tag: ChainEventType) !ChainEvent {
         .new_head => blk: {
             const proto_block = types.ProtoBlock{
                 .slot = 999_999,
+                .proposer_index = 0,
                 .blockRoot = [_]u8{0xab} ** 32,
                 .parentRoot = [_]u8{0xcd} ** 32,
                 .stateRoot = [_]u8{0xef} ** 32,

pkgs/cli/src/main.zig

@@ -19,7 +19,7 @@ const node_lib = @import("@zeam/node");
 const Clock = node_lib.Clock;
 const state_proving_manager = @import("@zeam/state-proving-manager");
 const BeamNode = node_lib.BeamNode;
-const xev = @import("xev");
+const xev = @import("xev").Dynamic;
 const Multiaddr = @import("multiaddr").Multiaddr;
 
 const configs = @import("@zeam/configs");
@@ -28,6 +28,7 @@ const Chain = configs.Chain;
 const ChainOptions = configs.ChainOptions;
 
 const utils_lib = @import("@zeam/utils");
+const key_manager_lib = @import("@zeam/key-manager");
 const zeam_metrics = @import("@zeam/metrics");
 
 const database = @import("@zeam/database");
@@ -70,6 +71,8 @@ pub const NodeCommand = struct {
     @"data-dir": []const u8 = constants.DEFAULT_DATA_DIR,
     @"checkpoint-sync-url": ?[]const u8 = null,
     @"is-aggregator": bool = false,
+    @"attestation-committee-count": ?u64 = null,
+    @"aggregate-subnet-ids": ?[]const u8 = null,
 
     pub const __shorts__ = .{
         .help = .h,
@@ -90,6 +93,8 @@ pub const NodeCommand = struct {
         .@"data-dir" = "Path to the data directory",
         .@"checkpoint-sync-url" = "URL to fetch finalized checkpoint state from for checkpoint sync (e.g., http://localhost:5052/lean/v0/states/finalized)",
         .@"is-aggregator" = "Enable aggregator mode for committee signature aggregation",
+        .@"attestation-committee-count" = "Number of attestation committees (subnets); overrides config.yaml ATTESTATION_COMMITTEE_COUNT",
+        .@"aggregate-subnet-ids" = "Comma-separated list of subnet ids to additionally subscribe and aggregate gossip attestations (e.g. '0,1,2'); adds to automatic computation from validator ids",
         .help = "Show help information for the node command",
     };
 };
@@ -276,6 +281,9 @@ fn mainInner() !void {
 
     std.debug.print("opts={any} genesis={d}\n", .{ opts.args, genesis });
 
+    // Detect the best available I/O backend (io_uring or epoll on Linux).
+    node_lib.detectBackend();
+
     switch (opts.args.__commands__) {
         .clock => {
             var loop = xev.Loop.init(.{}) catch |err| {
@@ -396,7 +404,6 @@ fn mainInner() !void {
             var chain_options = (try json.parseFromSlice(ChainOptions, gpa.allocator(), chain_spec, options)).value;
 
             // Create key manager FIRST to get validator pubkeys for genesis
-            const key_manager_lib = @import("@zeam/key-manager");
             // Using 3 validators for 3-node setup with initial sync testing
             // Nodes 1,2 start immediately; Node 3 starts after finalization to test sync
             const num_validators: usize = 3;
@@ -484,7 +491,7 @@ fn mainInner() !void {
                 backend1 = network.getNetworkInterface();
                 backend2 = network.getNetworkInterface();
                 backend3 = network.getNetworkInterface();
-                logger1_config.logger(null).debug("--- mock gossip {any}", .{backend1.gossip});
+                logger1_config.logger(null).debug("--- mock gossip {f}", .{backend1.gossip});
             } else {
                 network1 = try allocator.create(networks.EthLibp2p);
                 const key_pair1 = enr_lib.KeyPair.generate();
@@ -554,7 +561,7 @@ fn mainInner() !void {
                     .attestation_committee_count = chain_config.spec.attestation_committee_count,
                 }, logger3_config.logger(.network));
                 backend3 = network3.getNetworkInterface();
-                logger1_config.logger(null).debug("--- ethlibp2p gossip {any}", .{backend1.gossip});
+                logger1_config.logger(null).debug("--- ethlibp2p gossip {f}", .{backend1.gossip});
             }
 
             var clock = try allocator.create(Clock);
@@ -794,30 +801,8 @@ fn mainInner() !void {
                 };
                 defer allocator.free(sk_path);
 
-                const pk_file = std.fs.cwd().openFile(key_path, .{}) catch |err| {
-                    ErrorHandler.logErrorWithDetails(err, "open public key file", .{ .path = key_path });
-                    return err;
-                };
-                defer pk_file.close();
-                const pk_bytes = pk_file.readToEndAlloc(allocator, 256) catch |err| {
-                    ErrorHandler.logErrorWithOperation(err, "read public key file");
-                    return err;
-                };
-                defer allocator.free(pk_bytes);
-
-                const sk_file = std.fs.cwd().openFile(sk_path, .{}) catch |err| {
-                    ErrorHandler.logErrorWithDetails(err, "open private key file", .{ .path = sk_path });
-                    return err;
-                };
-                defer sk_file.close();
-                const sk_bytes = sk_file.readToEndAlloc(allocator, 16 * 1024 * 1024) catch |err| {
-                    ErrorHandler.logErrorWithOperation(err, "read private key file");
-                    return err;
-                };
-                defer allocator.free(sk_bytes);
-
-                keypair = xmss.KeyPair.fromSsz(allocator, sk_bytes, pk_bytes) catch |err| {
-                    ErrorHandler.logErrorWithOperation(err, "load keypair from SSZ");
+                keypair = key_manager_lib.loadKeypairFromFiles(allocator, sk_path, key_path) catch |err| {
+                    ErrorHandler.logErrorWithOperation(err, "load keypair from SSZ files");
                     return err;
                 };
             } else if (cmd.@"private-key") |seed| {