hash-zig integration

.github/workflows/ci.yml

@@ -232,10 +232,13 @@ jobs:
         exit 1
 
     - name: Run all unit tests
-      run: zig build test --summary all
+      run: zig build test -Doptimize=ReleaseFast --summary all
 
-    - name: Run all sim tests
-      run: zig build simtest --summary all
+    - name: Build zeam CLI for integration tests
+      run: zig build -Doptimize=ReleaseSafe
+
+    - name: Run integration tests (with instrumentation)
+      run: timeout 600 zig build simtest -Doptimize=ReleaseSafe --summary all
 
     - name: Install uv
       shell: bash

build.zig

@@ -22,23 +22,23 @@ fn addRustGlueLib(b: *Builder, comp: *Builder.Step.Compile, target: Builder.Reso
     // Use profile-specific directories for single-prover builds
     switch (prover) {
         .dummy => {
-            comp.addObjectFile(b.path("rust/target/release/libhashsig_glue.a"));
+            // hashsig-glue removed - now using pure Zig hash-zig
             comp.addObjectFile(b.path("rust/target/release/liblibp2p_glue.a"));
         },
         .risc0 => {
             comp.addObjectFile(b.path("rust/target/risc0-release/librisc0_glue.a"));
-            comp.addObjectFile(b.path("rust/target/risc0-release/libhashsig_glue.a"));
+            // hashsig-glue removed - now using pure Zig hash-zig
             comp.addObjectFile(b.path("rust/target/risc0-release/liblibp2p_glue.a"));
         },
         .openvm => {
             comp.addObjectFile(b.path("rust/target/openvm-release/libopenvm_glue.a"));
-            comp.addObjectFile(b.path("rust/target/openvm-release/libhashsig_glue.a"));
+            // hashsig-glue removed - now using pure Zig hash-zig
             comp.addObjectFile(b.path("rust/target/openvm-release/liblibp2p_glue.a"));
         },
         .all => {
             comp.addObjectFile(b.path("rust/target/release/librisc0_glue.a"));
             comp.addObjectFile(b.path("rust/target/release/libopenvm_glue.a"));
-            comp.addObjectFile(b.path("rust/target/release/libhashsig_glue.a"));
+            // hashsig-glue removed - now using pure Zig hash-zig
             comp.addObjectFile(b.path("rust/target/release/liblibp2p_glue.a"));
         },
     }
@@ -188,12 +188,20 @@ pub fn build(b: *Builder) !void {
     zeam_api.addImport("@zeam/types", zeam_types);
     zeam_api.addImport("@zeam/utils", zeam_utils);
 
+    // add hash-zig dependency
+    const hash_zig_dep = b.dependency("hash-zig", .{
+        .target = target,
+        .optimize = optimize,
+    });
+    const hash_zig = hash_zig_dep.module("hash-zig");
+
     // add zeam-xmss
     const zeam_xmss = b.addModule("@zeam/xmss", .{
         .target = target,
         .optimize = optimize,
         .root_source_file = b.path("pkgs/xmss/src/hashsig.zig"),
     });
+    zeam_xmss.addImport("hash-zig", hash_zig);
 
     // add zeam-key-manager
     const zeam_key_manager = b.addModule("@zeam/key-manager", .{
@@ -638,17 +646,17 @@ fn build_rust_project(b: *Builder, path: []const u8, prover: ProverChoice) *Buil
     const cargo_build = switch (prover) {
         .dummy => b.addSystemCommand(&.{
             "cargo", "+nightly",  "-C", path,          "-Z", "unstable-options",
-            "build", "--release", "-p", "libp2p-glue", "-p", "hashsig-glue",
+            "build", "--release", "-p", "libp2p-glue",
         }),
         .risc0 => b.addSystemCommand(&.{
             "cargo",      "+nightly",  "-C",            path, "-Z",          "unstable-options",
             "build",      "--profile", "risc0-release", "-p", "libp2p-glue", "-p",
-            "risc0-glue", "-p",        "hashsig-glue",
+            "risc0-glue",
         }),
         .openvm => b.addSystemCommand(&.{
             "cargo",       "+nightly",  "-C",             path, "-Z",          "unstable-options",
             "build",       "--profile", "openvm-release", "-p", "libp2p-glue", "-p",
-            "openvm-glue", "-p",        "hashsig-glue",
+            "openvm-glue",
         }),
         .all => b.addSystemCommand(&.{
             "cargo", "+nightly",  "-C",    path, "-Z", "unstable-options",

build.zig.zon

@@ -43,6 +43,10 @@
             .url = "https://github.com/blockblaz/snappyframesz/archive/78e66c2d42dc44dd1177ad007e28838a82157aa3.tar.gz",
             .hash = "snappyframesz-0.0.1-COCLy9EQBADDWj8BS-OdrIFOwHfiY9KUUiClsyBHgETn",
         },
+        .@"hash-zig" = .{
+            .url = "https://github.com/blockblaz/hash-zig/archive/refs/tags/v1.1.3.tar.gz",
+            .hash = "1220b0dc5fe3420bc478ddc2892885b71a070a9e1e9ba9ae815b66bd0963c5145b83",
+        },
     },
     .paths = .{""},
 }

pkgs/cli/src/main.zig

@@ -325,7 +325,9 @@ fn mainInner() !void {
             const key_manager_lib = @import("@zeam/key-manager");
             // Using 3 validators: so by default beam cmd command runs two nodes to interop
             const num_validators: usize = 3;
-            var key_manager = try key_manager_lib.getTestKeyManager(allocator, num_validators, 1000);
+            // Use 100 max_slot for beam command (will be rounded up to 1024 minimum)
+            // This is sufficient for integration tests while keeping key generation reasonable
+            var key_manager = try key_manager_lib.getTestKeyManager(allocator, num_validators, 100);
             defer key_manager.deinit();
 
             // Get validator pubkeys from keymanager

pkgs/cli/src/node.zig

@@ -157,6 +157,9 @@ pub const Node = struct {
         self.key_manager = key_manager_lib.KeyManager.init(allocator);
         errdefer self.key_manager.deinit();
 
+        // Initialize logger BEFORE loadValidatorKeypairs so it can log
+        self.logger = options.logger_config.logger(.node);
+
         try self.loadValidatorKeypairs(num_validators);
 
         try self.beam_node.init(allocator, .{
@@ -170,8 +173,6 @@ pub const Node = struct {
             .db = db,
             .logger_config = options.logger_config,
         });
-
-        self.logger = options.logger_config.logger(.node);
     }
 
     pub fn deinit(self: *Self) void {
@@ -317,33 +318,67 @@ pub const Node = struct {
                 return error.HashSigValidatorIndexOutOfRange;
             }
 
-            const pk_path = try std.fmt.allocPrint(self.allocator, "{s}/validator_{d}_pk.json", .{ hash_sig_key_dir, validator_index });
-            defer self.allocator.free(pk_path);
+            // Try SSZ format first (preferred), then fall back to JSON
+            const pk_ssz_path = try std.fmt.allocPrint(self.allocator, "{s}/validator_{d}_pk.ssz", .{ hash_sig_key_dir, validator_index });
+            defer self.allocator.free(pk_ssz_path);
+            const sk_ssz_path = try std.fmt.allocPrint(self.allocator, "{s}/validator_{d}_sk.ssz", .{ hash_sig_key_dir, validator_index });
+            defer self.allocator.free(sk_ssz_path);
 
-            var pk_file = std.fs.cwd().openFile(pk_path, .{}) catch |err| switch (err) {
-                error.FileNotFound => return error.HashSigPublicKeyMissing,
-                else => return err,
-            };
-            defer pk_file.close();
-            const public_json = try pk_file.readToEndAlloc(self.allocator, constants.MAX_HASH_SIG_KEY_JSON_SIZE);
-            defer self.allocator.free(public_json);
+            self.logger.info("Loading hash-sig keys for validator {d}: pk={s}, sk={s}", .{ validator_index, pk_ssz_path, sk_ssz_path });
 
-            const sk_path = try std.fmt.allocPrint(self.allocator, "{s}/validator_{d}_sk.json", .{ hash_sig_key_dir, validator_index });
-            defer self.allocator.free(sk_path);
+            // Check if SSZ files exist
+            const ssz_exists = blk: {
+                std.fs.cwd().access(pk_ssz_path, .{}) catch break :blk false;
+                std.fs.cwd().access(sk_ssz_path, .{}) catch break :blk false;
+                break :blk true;
+            };
 
-            var sk_file = std.fs.cwd().openFile(sk_path, .{}) catch |err| switch (err) {
-                error.FileNotFound => return error.HashSigSecretKeyMissing,
-                else => return err,
+            var keypair = if (ssz_exists) blk: {
+                // Load SSZ format
+                var pk_file = try std.fs.cwd().openFile(pk_ssz_path, .{});
+                defer pk_file.close();
+                const public_ssz = try pk_file.readToEndAlloc(self.allocator, constants.MAX_HASH_SIG_KEY_JSON_SIZE);
+                defer self.allocator.free(public_ssz);
+
+                var sk_file = try std.fs.cwd().openFile(sk_ssz_path, .{});
+                defer sk_file.close();
+                const secret_ssz = try sk_file.readToEndAlloc(self.allocator, constants.MAX_HASH_SIG_KEY_JSON_SIZE);
+                defer self.allocator.free(secret_ssz);
+
+                break :blk try xmss.KeyPair.fromSSZ(
+                    self.allocator,
+                    secret_ssz,
+                    public_ssz,
+                );
+            } else blk: {
+                // Fall back to JSON format
+                const pk_json_path = try std.fmt.allocPrint(self.allocator, "{s}/validator_{d}_pk.json", .{ hash_sig_key_dir, validator_index });
+                defer self.allocator.free(pk_json_path);
+                const sk_json_path = try std.fmt.allocPrint(self.allocator, "{s}/validator_{d}_sk.json", .{ hash_sig_key_dir, validator_index });
+                defer self.allocator.free(sk_json_path);
+
+                var pk_file = std.fs.cwd().openFile(pk_json_path, .{}) catch |err| switch (err) {
+                    error.FileNotFound => return error.HashSigPublicKeyMissing,
+                    else => return err,
+                };
+                defer pk_file.close();
+                const public_json = try pk_file.readToEndAlloc(self.allocator, constants.MAX_HASH_SIG_KEY_JSON_SIZE);
+                defer self.allocator.free(public_json);
+
+                var sk_file = std.fs.cwd().openFile(sk_json_path, .{}) catch |err| switch (err) {
+                    error.FileNotFound => return error.HashSigSecretKeyMissing,
+                    else => return err,
+                };
+                defer sk_file.close();
+                const secret_json = try sk_file.readToEndAlloc(self.allocator, constants.MAX_HASH_SIG_KEY_JSON_SIZE);
+                defer self.allocator.free(secret_json);
+
+                break :blk try xmss.KeyPair.fromJson(
+                    self.allocator,
+                    secret_json,
+                    public_json,
+                );
             };
-            defer sk_file.close();
-            const secret_json = try sk_file.readToEndAlloc(self.allocator, constants.MAX_HASH_SIG_KEY_JSON_SIZE);
-            defer self.allocator.free(secret_json);
-
-            var keypair = try xmss.KeyPair.fromJson(
-                self.allocator,
-                secret_json,
-                public_json,
-            );
             errdefer keypair.deinit();
 
             try self.key_manager.addKeypair(validator_index, keypair);

pkgs/cli/test/integration.zig

@@ -488,18 +488,35 @@ test "SSE events integration test - wait for justification and finalization" {
     try sse_client.connect();
 
     std.debug.print("INFO: Connected to SSE endpoint, waiting for events...\n", .{});
+    const test_start_time = std.time.milliTimestamp();
+    std.debug.print("INFO: Test started at timestamp {}\n", .{test_start_time});
 
     // Read events until both justification and finalization are seen, or timeout
-    const timeout_ms: u64 = 180000; // 180 seconds timeout
+    const timeout_ms: u64 = 480_000; // 480 seconds (8 minutes)
     const start_ns = std.time.nanoTimestamp();
     const deadline_ns = start_ns + timeout_ms * std.time.ns_per_ms;
     var got_justification = false;
     var got_finalization = false;
+    var event_count: usize = 0;
+    var last_progress_log = std.time.milliTimestamp();
 
     // FIXED: This loop now works correctly with the improved readEvent() function
     while (std.time.nanoTimestamp() < deadline_ns and !(got_justification and got_finalization)) {
+        // Log progress every 30 seconds
+        const now = std.time.milliTimestamp();
+        if (now - last_progress_log > 30000) {
+            const elapsed_sec = @divTrunc(now - test_start_time, 1000);
+            std.debug.print("INFO: Still waiting... {}s elapsed, {} events received, justification={}, finalization={}\n", .{
+                elapsed_sec,
+                event_count,
+                got_justification,
+                got_finalization,
+            });
+            last_progress_log = now;
+        }
         const event = try sse_client.readEvent();
         if (event) |e| {
+            event_count += 1;
             // Check for justification with slot > 0
             if (!got_justification and std.mem.eql(u8, e.event_type, "new_justification")) {
                 if (e.justified_slot) |slot| {
@@ -538,7 +555,7 @@ test "SSE events integration test - wait for justification and finalization" {
 
     std.debug.print("INFO: Received events - Head: {}, Justification: {}, Finalization: {}\n", .{ head_events, justification_events, finalization_events });
 
-    // Require both justification and finalization (> 0) to have been observed
+    // Require both justification and finalization (timeout extended to 6 minutes)
     try std.testing.expect(got_justification);
     try std.testing.expect(got_finalization);
 

pkgs/key-manager/src/lib.zig

@@ -6,19 +6,24 @@ const Allocator = std.mem.Allocator;
 
 const KeyManagerError = error{
     ValidatorKeyNotFound,
+    SignatureMismatch,
 };
 
 const CachedKeyPair = struct {
     keypair: xmss.KeyPair,
     num_active_epochs: usize,
 };
 var global_test_key_pair_cache: ?std.AutoHashMap(usize, CachedKeyPair) = null;
+var cache_mutex: std.Thread.Mutex = .{};
 const cache_allocator = std.heap.page_allocator;
 
 fn getOrCreateCachedKeyPair(
     validator_id: usize,
     num_active_epochs: usize,
 ) !xmss.KeyPair {
+    cache_mutex.lock();
+    defer cache_mutex.unlock();
+
     if (global_test_key_pair_cache == null) {
         global_test_key_pair_cache = std.AutoHashMap(usize, CachedKeyPair).init(cache_allocator);
     }
@@ -106,6 +111,8 @@ pub const KeyManager = struct {
 
         if (bytes_written < types.SIGSIZE) {
             @memset(sig_buffer[bytes_written..], 0);
+        } else if (bytes_written > types.SIGSIZE) {
+            return KeyManagerError.SignatureMismatch;
         }
 
         return sig_buffer;
@@ -149,14 +156,66 @@ pub fn getTestKeyManager(
     errdefer key_manager.deinit();
 
     var num_active_epochs = max_slot + 1;
-    // to reuse cached keypairs, gen for 10 since most tests ask for < 10 max slot including
-    // building mock chain for tests. otherwise getOrCreateCachedKeyPair might cleanup previous
-    //  key generated for smaller life time
+    // For tests, use minimum of 10 epochs
     if (num_active_epochs < 10) num_active_epochs = 10;
 
-    for (0..num_validators) |i| {
-        const keypair = try getOrCreateCachedKeyPair(i, num_active_epochs);
-        try key_manager.addKeypair(i, keypair);
+    // Parallelize key generation for multiple validators
+    if (num_validators > 1) {
+        // Create threads for parallel key generation
+        const KeyGenContext = struct {
+            validator_id: usize,
+            num_active_epochs: usize,
+            result: ?xmss.KeyPair = null,
+            err: ?anyerror = null,
+        };
+
+        var contexts = try allocator.alloc(KeyGenContext, num_validators);
+        defer allocator.free(contexts);
+
+        for (contexts, 0..) |*ctx, i| {
+            ctx.* = KeyGenContext{
+                .validator_id = i,
+                .num_active_epochs = num_active_epochs,
+            };
+        }
+
+        const threads = try allocator.alloc(std.Thread, num_validators);
+        defer allocator.free(threads);
+
+        // Spawn threads for parallel key generation
+        for (threads, 0..) |*thread, i| {
+            thread.* = try std.Thread.spawn(.{}, struct {
+                fn run(ctx: *KeyGenContext) void {
+                    ctx.result = getOrCreateCachedKeyPair(ctx.validator_id, ctx.num_active_epochs) catch |err| {
+                        ctx.err = err;
+                        return;
+                    };
+                }
+            }.run, .{&contexts[i]});
+        }
+
+        // Wait for all threads to complete
+        for (threads) |thread| {
+            thread.join();
+        }
+
+        // Collect results and check for errors
+        for (contexts) |ctx| {
+            if (ctx.err) |err| {
+                return err;
+            }
+            if (ctx.result) |keypair| {
+                try key_manager.addKeypair(ctx.validator_id, keypair);
+            } else {
+                return error.KeyGenerationFailed;
+            }
+        }
+    } else {
+        // Single validator - no need for parallelization
+        for (0..num_validators) |i| {
+            const keypair = try getOrCreateCachedKeyPair(i, num_active_epochs);
+            try key_manager.addKeypair(i, keypair);
+        }
     }
 
     return key_manager;

pkgs/types/src/state.zig

@@ -406,11 +406,16 @@ pub const BeamState = struct {
                 logger.debug("\n\n\n-----------------HURRAY JUSTIFICATION ------------\n{s}\n--------------\n---------------\n-------------------------\n\n\n", .{justified_str_new});
 
                 // source is finalized if target is the next valid justifiable hash
+                // Special case: allow genesis (slot 0) to finalize when any slot is justified,
+                // since slots 1-5 are always justifiable from genesis, which would prevent
+                // any finalization from ever happening.
                 var can_target_finalize = true;
-                for (source_slot + 1..target_slot) |check_slot| {
-                    if (try utils.IsJustifiableSlot(self.latest_finalized.slot, check_slot)) {
-                        can_target_finalize = false;
-                        break;
+                if (self.latest_finalized.slot > 0) {
+                    for (source_slot + 1..target_slot) |check_slot| {
+                        if (try utils.IsJustifiableSlot(self.latest_finalized.slot, check_slot)) {
+                            can_target_finalize = false;
+                            break;
+                        }
                     }
                 }
                 logger.debug("----------------can_target_finalize ({d})={any}----------\n\n", .{ source_slot, can_target_finalize });