diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ab60319c..cae157925 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: recursive - name: Set up Zig uses: mlugg/setup-zig@v2.0.5 @@ -82,6 +84,8 @@ jobs: os: [ubuntu-latest, macos-latest] steps: - uses: actions/checkout@v4 + with: + submodules: recursive - name: Free disk space (Ubuntu) if: runner.os == 'Linux' @@ -142,6 +146,8 @@ jobs: os: [ubuntu-latest, macos-latest] steps: - uses: actions/checkout@v4 + with: + submodules: recursive - name: Free disk space (Ubuntu) if: runner.os == 'Linux' @@ -282,6 +288,8 @@ jobs: os: [ubuntu-latest, macos-latest] steps: - uses: actions/checkout@v4 + with: + submodules: recursive - name: Free disk space (Ubuntu) if: runner.os == 'Linux' @@ -327,6 +335,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + submodules: recursive fetch-depth: 0 # Fetch full history to get git commit info - name: Free disk space (Ubuntu) diff --git a/.gitmodules b/.gitmodules index fbc272db1..0c25c3085 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "leanSpec"] path = leanSpec url = https://github.com/leanEthereum/leanSpec +[submodule "test-keys"] + path = test-keys + url = https://github.com/blockblaz/zeam-test-keys.git diff --git a/build.zig b/build.zig index 43e2da5a6..588b0450e 100644 --- a/build.zig +++ b/build.zig @@ -156,6 +156,8 @@ pub fn build(b: *Builder) !void { build_options.addOption(bool, "has_openvm", prover == .openvm or prover == .all); const use_poseidon = b.option(bool, "use_poseidon", "Use Poseidon SSZ hasher (default: false)") orelse false; build_options.addOption(bool, "use_poseidon", use_poseidon); + // Absolute path to test-keys for pre-generated validator keys + build_options.addOption([]const u8, "test_keys_path", b.pathFromRoot("test-keys/hash-sig-keys")); const build_options_module = build_options.createModule(); // add zeam-utils @@ -240,6 +242,7 @@ pub fn build(b: *Builder) !void { .target = target, .optimize = optimize, }); + zeam_key_manager.addImport("build_options", build_options_module); zeam_key_manager.addImport("@zeam/xmss", zeam_xmss); zeam_key_manager.addImport("@zeam/types", zeam_types); zeam_key_manager.addImport("@zeam/utils", zeam_utils); @@ -421,6 +424,10 @@ pub fn build(b: *Builder) !void { tools_cli_exe.root_module.addImport("enr", enr); tools_cli_exe.root_module.addImport("build_options", build_options_module); tools_cli_exe.root_module.addImport("simargs", simargs); + tools_cli_exe.root_module.addImport("@zeam/xmss", zeam_xmss); + tools_cli_exe.root_module.addImport("@zeam/types", zeam_types); + tools_cli_exe.step.dependOn(&build_rust_lib_steps.step); + addRustGlueLib(b, tools_cli_exe, target, prover); const install_tools_cli = b.addInstallArtifact(tools_cli_exe, .{}); tools_step.dependOn(&install_tools_cli.step); @@ -602,6 +609,10 @@ pub fn build(b: *Builder) !void { .root_module = tools_cli_exe.root_module, }); tools_cli_tests.root_module.addImport("enr", enr); + tools_cli_tests.root_module.addImport("@zeam/xmss", zeam_xmss); + tools_cli_tests.root_module.addImport("@zeam/types", zeam_types); + tools_cli_tests.step.dependOn(&build_rust_lib_steps.step); + addRustGlueLib(b, tools_cli_tests, target, prover); const run_tools_cli_test = b.addRunArtifact(tools_cli_tests); setTestRunLabelFromCompile(b, run_tools_cli_test, tools_cli_tests); tools_test_step.dependOn(&run_tools_cli_test.step); diff --git a/pkgs/key-manager/src/lib.zig b/pkgs/key-manager/src/lib.zig index ce3e933d3..33c7f326b 100644 --- a/pkgs/key-manager/src/lib.zig +++ b/pkgs/key-manager/src/lib.zig @@ -55,7 +55,8 @@ fn getOrCreateCachedKeyPair( pub const KeyManager = struct { keys: std.AutoHashMap(usize, xmss.KeyPair), allocator: Allocator, - owns_keypairs: bool, + /// Tracks which keypairs are owned (allocated by us) vs borrowed (cached). + owned_keys: std.AutoHashMap(usize, void), const Self = @This(); @@ -63,22 +64,30 @@ pub const KeyManager = struct { return Self{ .keys = std.AutoHashMap(usize, xmss.KeyPair).init(allocator), .allocator = allocator, - .owns_keypairs = true, + .owned_keys = std.AutoHashMap(usize, void).init(allocator), }; } pub fn deinit(self: *Self) void { - if (self.owns_keypairs) { - var it = self.keys.iterator(); - while (it.next()) |entry| { + var it = self.keys.iterator(); + while (it.next()) |entry| { + if (self.owned_keys.contains(entry.key_ptr.*)) { entry.value_ptr.deinit(); } } self.keys.deinit(); + self.owned_keys.deinit(); } + /// Add an owned keypair that will be freed on deinit. pub fn addKeypair(self: *Self, validator_id: usize, keypair: xmss.KeyPair) !void { try self.keys.put(validator_id, keypair); + try self.owned_keys.put(validator_id, {}); + } + + /// Add a cached/borrowed keypair that will NOT be freed on deinit. + pub fn addCachedKeypair(self: *Self, validator_id: usize, keypair: xmss.KeyPair) !void { + try self.keys.put(validator_id, keypair); } pub fn loadFromKeypairDir(_: *Self, _: []const u8) !void { @@ -162,24 +171,105 @@ pub const KeyManager = struct { } }; +/// Maximum size of a serialized XMSS private key (20MB). +const MAX_SK_SIZE = 1024 * 1024 * 20; + +/// Maximum size of a serialized XMSS public key (256 bytes). +const MAX_PK_SIZE = 256; + +/// Number of pre-generated test keys available in the test-keys submodule. +const NUM_PREGENERATED_KEYS: usize = 32; + +const build_options = @import("build_options"); + +/// Find the test-keys directory using the repo root path injected by build.zig. +fn findTestKeysDir() ?[]const u8 { + const keys_path = build_options.test_keys_path; + if (keys_path.len == 0) return null; + + // Verify it actually exists at runtime + if (std.fs.cwd().openDir(keys_path, .{})) |dir| { + var d = dir; + d.close(); + return keys_path; + } else |_| {} + + return null; +} + +/// Load a single pre-generated key pair from SSZ files on disk. +fn loadPreGeneratedKey( + allocator: Allocator, + keys_dir: []const u8, + index: usize, +) !xmss.KeyPair { + // Build file paths + var sk_path_buf: [512]u8 = undefined; + const sk_path = std.fmt.bufPrint(&sk_path_buf, "{s}/validator_{d}_sk.ssz", .{ keys_dir, index }) catch unreachable; + + var pk_path_buf: [512]u8 = undefined; + const pk_path = std.fmt.bufPrint(&pk_path_buf, "{s}/validator_{d}_pk.ssz", .{ keys_dir, index }) catch unreachable; + + // Read private key + var sk_file = try std.fs.cwd().openFile(sk_path, .{}); + defer sk_file.close(); + const sk_data = try sk_file.readToEndAlloc(allocator, MAX_SK_SIZE); + defer allocator.free(sk_data); + + // Read public key + var pk_file = try std.fs.cwd().openFile(pk_path, .{}); + defer pk_file.close(); + const pk_data = try pk_file.readToEndAlloc(allocator, MAX_PK_SIZE); + defer allocator.free(pk_data); + + // Reconstruct keypair from SSZ + return xmss.KeyPair.fromSsz(allocator, sk_data, pk_data); +} + pub fn getTestKeyManager( allocator: Allocator, num_validators: usize, max_slot: usize, ) !KeyManager { var key_manager = KeyManager.init(allocator); - key_manager.owns_keypairs = false; errdefer key_manager.deinit(); - var num_active_epochs = max_slot + 1; - // to reuse cached keypairs, gen for 10 since most tests ask for < 10 max slot including - // building mock chain for tests. otherwise getOrCreateCachedKeyPair might cleanup previous - // key generated for smaller life time - if (num_active_epochs < 10) num_active_epochs = 10; + // Determine how many keys we can load from pre-generated files + const keys_dir = findTestKeysDir(); + const num_preloaded = if (keys_dir != null) + @min(num_validators, NUM_PREGENERATED_KEYS) + else + 0; + + // Load pre-generated keys (fast path: near-instant from SSZ files) + var actually_loaded: usize = 0; + if (keys_dir) |dir| { + for (0..num_preloaded) |i| { + const keypair = loadPreGeneratedKey(allocator, dir, i) catch |err| { + std.debug.print("Failed to load pre-generated key {d}: {}\n", .{ i, err }); + break; + }; + key_manager.addKeypair(i, keypair) catch |err| { + std.debug.print("Failed to add pre-generated key {d}: {}\n", .{ i, err }); + break; + }; + actually_loaded += 1; + } + std.debug.print("Loaded {d} pre-generated test keys from {s}\n", .{ actually_loaded, dir }); + } else { + std.debug.print("Pre-generated keys not found, generating all keys at runtime\n", .{}); + } + + // Generate remaining keys at runtime (for validators beyond the loaded set) + if (num_validators > actually_loaded) { + var num_active_epochs = max_slot + 1; + if (num_active_epochs < 10) num_active_epochs = 10; - for (0..num_validators) |i| { - const keypair = try getOrCreateCachedKeyPair(i, num_active_epochs); - try key_manager.addKeypair(i, keypair); + for (actually_loaded..num_validators) |i| { + const keypair = try getOrCreateCachedKeyPair(i, num_active_epochs); + try key_manager.addCachedKeypair(i, keypair); + } + std.debug.print("Generated {d} additional keys at runtime\n", .{num_validators - actually_loaded}); } return key_manager; diff --git a/pkgs/tools/src/main.zig b/pkgs/tools/src/main.zig index 44d52fd66..43b9c25f9 100644 --- a/pkgs/tools/src/main.zig +++ b/pkgs/tools/src/main.zig @@ -2,6 +2,7 @@ const std = @import("std"); const enr = @import("enr"); const build_options = @import("build_options"); const simargs = @import("simargs"); +const xmss = @import("@zeam/xmss"); pub const max_enr_txt_size = enr.max_enr_txt_size; @@ -11,9 +12,11 @@ const ToolsArgs = struct { __commands__: union(enum) { enrgen: ENRGenCmd, + keygen: KeyGenCmd, pub const __messages__ = .{ .enrgen = "Generate a new ENR (Ethereum Node Record)", + .keygen = "Generate pre-computed XMSS test validator keys", }; }, @@ -50,6 +53,27 @@ const ToolsArgs = struct { .help = "Show help information for the enrgen command", }; }; + + const KeyGenCmd = struct { + @"num-validators": usize = 32, + @"num-active-epochs": usize = 1000, + @"output-dir": []const u8 = "test-keys", + help: bool = false, + + pub const __shorts__ = .{ + .@"num-validators" = .n, + .@"num-active-epochs" = .e, + .@"output-dir" = .o, + .help = .h, + }; + + pub const __messages__ = .{ + .@"num-validators" = "Number of validator key pairs to generate (default: 32)", + .@"num-active-epochs" = "Number of active epochs for each key (default: 1000)", + .@"output-dir" = "Output directory for generated keys (default: test-keys)", + .help = "Show help information for the keygen command", + }; + }; }; pub fn main() !void { @@ -86,6 +110,12 @@ pub fn main() !void { defer enr.deinitGlobalSecp256k1Ctx(); switch (opts.args.__commands__) { + .keygen => |cmd| { + handleKeyGen(allocator, cmd) catch |err| { + std.debug.print("Error generating keys: {}\n", .{err}); + std.process.exit(1); + }; + }, .enrgen => |cmd| { handleENRGen(cmd) catch |err| switch (err) { error.EmptySecretKey => { @@ -113,6 +143,119 @@ pub fn main() !void { } } +fn handleKeyGen(allocator: std.mem.Allocator, cmd: ToolsArgs.KeyGenCmd) !void { + const num_validators = cmd.@"num-validators"; + const num_active_epochs = cmd.@"num-active-epochs"; + const output_dir = cmd.@"output-dir"; + + std.debug.print("Generating {d} validator keys with {d} active epochs...\n", .{ num_validators, num_active_epochs }); + std.debug.print("Output directory: {s}\n", .{output_dir}); + + // Create output directories + const hash_sig_dir = try std.fmt.allocPrint(allocator, "{s}/hash-sig-keys", .{output_dir}); + defer allocator.free(hash_sig_dir); + + std.fs.cwd().makePath(hash_sig_dir) catch |err| { + std.debug.print("Error creating directory {s}: {}\n", .{ hash_sig_dir, err }); + return err; + }; + + // Allocate buffers for serialization + // Private keys can be very large (~5-10MB for XMSS) + const sk_buffer = try allocator.alloc(u8, 1024 * 1024 * 20); // 20MB + defer allocator.free(sk_buffer); + var pk_buffer: [256]u8 = undefined; + + // Open manifest file + const manifest_path = try std.fmt.allocPrint(allocator, "{s}/validator-keys-manifest.yaml", .{output_dir}); + defer allocator.free(manifest_path); + const manifest_file = try std.fs.cwd().createFile(manifest_path, .{}); + defer manifest_file.close(); + + var manifest_buf: [4096]u8 = undefined; + var manifest_writer = manifest_file.writer(&manifest_buf); + + // Write manifest header + try manifest_writer.interface.print( + \\key_scheme: SIGTopLevelTargetSumLifetime32Dim64Base8 + \\hash_function: Poseidon2 + \\encoding: TargetSum + \\lifetime: {d} + \\num_active_epochs: {d} + \\num_validators: {d} + \\validators: + \\ + , .{ num_active_epochs, num_active_epochs, num_validators }); + + for (0..num_validators) |i| { + std.debug.print(" Generating validator {d}/{d}...\n", .{ i + 1, num_validators }); + + // Generate keypair with deterministic seed + const seed = try std.fmt.allocPrint(allocator, "test_validator_{d}", .{i}); + defer allocator.free(seed); + + var keypair = try xmss.KeyPair.generate(allocator, seed, 0, num_active_epochs); + defer keypair.deinit(); + + // Serialize public key + const pk_len = try keypair.pubkeyToBytes(&pk_buffer); + + // Serialize private key + const sk_len = try keypair.privkeyToBytes(sk_buffer); + + std.debug.print(" PK size: {d} bytes, SK size: {d} bytes\n", .{ pk_len, sk_len }); + + // Write private key file + const sk_filename = try std.fmt.allocPrint(allocator, "validator_{d}_sk.ssz", .{i}); + defer allocator.free(sk_filename); + const sk_path = try std.fmt.allocPrint(allocator, "{s}/{s}", .{ hash_sig_dir, sk_filename }); + defer allocator.free(sk_path); + + const sk_file = try std.fs.cwd().createFile(sk_path, .{}); + defer sk_file.close(); + var sk_write_buf: [65536]u8 = undefined; + var sk_writer = sk_file.writer(&sk_write_buf); + try sk_writer.interface.writeAll(sk_buffer[0..sk_len]); + try sk_writer.interface.flush(); + + // Write public key file + const pk_filename = try std.fmt.allocPrint(allocator, "validator_{d}_pk.ssz", .{i}); + defer allocator.free(pk_filename); + const pk_path = try std.fmt.allocPrint(allocator, "{s}/{s}", .{ hash_sig_dir, pk_filename }); + defer allocator.free(pk_path); + + const pk_file = try std.fs.cwd().createFile(pk_path, .{}); + defer pk_file.close(); + var pk_write_buf: [4096]u8 = undefined; + var pk_writer = pk_file.writer(&pk_write_buf); + try pk_writer.interface.writeAll(pk_buffer[0..pk_len]); + try pk_writer.interface.flush(); + + // Write manifest entry with pubkey as hex + // Format pubkey bytes as hex string + var hex_buf: [512]u8 = undefined; + const hex_len = pk_len * 2; + for (pk_buffer[0..pk_len], 0..) |byte, j| { + const high = byte >> 4; + const low = byte & 0x0f; + hex_buf[j * 2] = if (high < 10) '0' + high else 'a' + high - 10; + hex_buf[j * 2 + 1] = if (low < 10) '0' + low else 'a' + low - 10; + } + + try manifest_writer.interface.print( + \\- index: {d} + \\ pubkey_hex: "0x{s}" + \\ privkey_file: {s} + \\ + , .{ i, hex_buf[0..hex_len], sk_filename }); + } + + try manifest_writer.interface.flush(); + + std.debug.print("\nDone! Generated {d} keys in {s}/\n", .{ num_validators, output_dir }); + std.debug.print("Manifest written to {s}\n", .{manifest_path}); +} + fn handleENRGen(cmd: ToolsArgs.ENRGenCmd) !void { if (cmd.sk.len == 0) { return error.EmptySecretKey; diff --git a/test-keys b/test-keys new file mode 160000 index 000000000..0b645ebd2 --- /dev/null +++ b/test-keys @@ -0,0 +1 @@ +Subproject commit 0b645ebd2302636330689de12afe3e4e8dfde3df