added basic benchmarker for the CairoProgramRunner running simple bootloader on cairo pies + script for deploying it

on many pies. also for run_and_prove

Author: Idanh-starkware

crates/cairo-program-runner-lib/Cargo.toml

@@ -3,6 +3,13 @@ name = "cairo-program-runner-lib"
 version = "0.1.0"
 edition = "2021"
 
+[[bin]]
+name = "benchmark_runner"
+path = "scripts/benchmark_runner.rs"
+
+[[bin]]
+name = "benchmark_run_and_prove"
+path = "scripts/benchmark_run_and_prove.rs"
 
 [dependencies]
 bincode.workspace = true

crates/cairo-program-runner-lib/scripts/benchmark_run_and_prove.rs

@@ -0,0 +1,149 @@
+// benchmark_run_and_prove.rs
+// Usage: cargo run --bin benchmark_run_and_prove <path_to_pie_file> [--show-output]
+//
+// This program benchmarks proving a Cairo PIE file using the simple bootloader as the program
+// and the PIE as the program input. It prints load/prove times and proof status.
+
+use std::error::Error;
+use std::fs::File;
+use std::io::Write;
+use std::path::PathBuf;
+use std::process::Command;
+use std::time::Instant;
+
+fn main() -> Result<(), Box<dyn Error>> {
+    let project_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    let simple_bootloader_compiled_path =
+        project_dir.join("resources/compiled_programs/bootloaders/simple_bootloader_compiled.json");
+
+    // Print and check for simple_bootloader_compiled.json
+    println!(
+        "Looking for simple_bootloader_compiled.json at: {:?}",
+        simple_bootloader_compiled_path
+    );
+    if !simple_bootloader_compiled_path.exists() {
+        eprintln!(
+            "ERROR: File does not exist: {:?}",
+            simple_bootloader_compiled_path
+        );
+    }
+
+    // Parse command line args
+    let args: Vec<String> = std::env::args().collect();
+    let mut pie_path = None;
+    let mut show_output = false;
+    for (i, arg) in args.iter().enumerate() {
+        if arg == "--show-output" {
+            show_output = true;
+        } else if i > 0 && !arg.starts_with("--") {
+            pie_path = Some(arg.clone());
+        }
+    }
+    let pie_path = match pie_path {
+        Some(p) => p,
+        None => {
+            eprintln!("Usage: {} <path_to_pie_file> [--show-output]", args[0]);
+            std::process::exit(1);
+        }
+    };
+
+    println!("Proving PIE: {}", pie_path);
+    let start = Instant::now();
+
+    // Build the correct program_input JSON for a PIE file
+    let program_input_json = format!(
+        r#"{{
+            "tasks": [
+                {{
+                    "type": "CairoPiePath",
+                    "path": "{}",
+                    "program_hash_function": "blake"
+                }}
+            ],
+            "single_page": true
+        }}"#,
+        pie_path
+    );
+    // Write the JSON to a temporary file
+    let tmp_input_path = "temp_pie_input.json";
+    let mut tmp_file = File::create(tmp_input_path)?;
+    tmp_file.write_all(program_input_json.as_bytes())?;
+
+    // Get absolute path to prover_params.json
+    let prover_params_path = project_dir
+        .parent()
+        .unwrap()
+        .join("stwo_run_and_prove/resources/prover_params.json");
+    println!(
+        "Looking for prover_params.json at: {:?}",
+        prover_params_path
+    );
+    if !prover_params_path.exists() {
+        eprintln!("ERROR: File does not exist: {:?}", prover_params_path);
+    }
+
+    // Use the full path to the stwo_run_and_prove binary in target/release (fix path: remove
+    // 'crates/')
+    let stwo_bin_path = project_dir
+        .parent()
+        .unwrap()
+        .parent()
+        .unwrap()
+        .join("target/release/stwo_run_and_prove");
+    println!(
+        "Using stwo_run_and_prove binary at: {}",
+        stwo_bin_path.display()
+    );
+    if !stwo_bin_path.exists() {
+        eprintln!(
+            "ERROR: stwo_run_and_prove binary not found at {}",
+            stwo_bin_path.display()
+        );
+        std::process::exit(1);
+    }
+
+    // Print the command being run for debugging (for display only)
+    println!(
+        "Running command: {} --program=\"{}\" --program_input=\"{}\" --proofs_dir=temp_proofs --proof-format=cairo-serde --verify --prover_params_json=\"{}\" --program_output=temp_outputs",
+        stwo_bin_path.display(),
+        simple_bootloader_compiled_path.display(),
+        tmp_input_path,
+        prover_params_path.display()
+    );
+
+    // Call stwo_run_and_prove binary with the bootloader as program and the PIE as program_input
+    let output = Command::new(&stwo_bin_path)
+        .arg(format!(
+            "--program={}",
+            simple_bootloader_compiled_path.display()
+        ))
+        .arg(format!("--program_input={}", tmp_input_path))
+        .arg("--proofs_dir=temp_proofs")
+        .arg("--proof-format=cairo-serde")
+        .arg("--verify")
+        .arg(format!(
+            "--prover_params_json={}",
+            prover_params_path.display()
+        ))
+        .arg("--program_output=temp_outputs")
+        .output()?;
+    let elapsed = start.elapsed();
+    if output.status.success() {
+        println!(
+            "SUCCESS: Proved {} in {:.3} seconds",
+            pie_path,
+            elapsed.as_secs_f64()
+        );
+        if show_output {
+            println!("stdout:\n{}", String::from_utf8_lossy(&output.stdout));
+        }
+    } else {
+        println!(
+            "ERROR: Failed to prove {} (exit code {:?})",
+            pie_path,
+            output.status.code()
+        );
+        println!("stderr:\n{}", String::from_utf8_lossy(&output.stderr));
+    }
+    Ok(())
+}

crates/cairo-program-runner-lib/scripts/benchmark_runner.rs

@@ -0,0 +1,161 @@
+//! Benchmark Runner for Cairo PIE Files
+//!
+//! This program measures the performance of loading and executing Cairo PIEs
+//! wrapped in simple bootloader and ran by the Cairo Program Runner.
+//!
+//! Usage:
+//!   cargo run --bin benchmark_runner <path_to_pie_file> [--show-output]
+//!
+//! Arguments:
+//!   <path_to_pie_file>  Path to the Cairo PIE file (.zip) to benchmark
+//!   --show-output       Optional flag to display program output
+//!
+//! Output:
+//!   - Program load time
+//!   - Program execution time
+//!   - Total runtime
+//!   - Execution resources (steps, memory holes, builtin counters)
+//!
+//! This binary is typically run through the run_all_pies.sh script, which provides
+//! additional features:
+//!   - Running multiple PIE files
+//!   - Testing different Rust optimization levels
+//!   - Aggregating results into a single file
+//!
+//! Example:
+//!   Direct usage:
+//!     cargo run --bin benchmark_runner path/to/program.zip
+//!   Via script:
+//!     ./scripts/run_all_pies.sh --opt-levels=3 --pie-dir=path/to/pies
+
+use std::error::Error;
+use std::path::PathBuf;
+use std::time::Instant;
+
+use cairo_program_runner_lib::cairo_run_program;
+use cairo_program_runner_lib::types::RunMode;
+use cairo_vm::types::layout_name::LayoutName;
+use cairo_vm::types::program::Program;
+use cairo_vm::vm::runners::cairo_pie::CairoPie;
+
+fn main() -> Result<(), Box<dyn Error>> {
+    let project_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    let simple_bootloader_compiled_path =
+        project_dir.join("resources/compiled_programs/bootloaders/simple_bootloader_compiled.json");
+
+    // Parse command line args
+    let args: Vec<String> = std::env::args().collect();
+    let mut program_path = String::new();
+    let mut show_output = false;
+
+    // Simple arg parsing
+    for (i, arg) in args.iter().enumerate() {
+        if arg == "--show-output" {
+            show_output = true;
+        } else if i > 0 && !arg.starts_with("--") {
+            program_path = arg.clone();
+        }
+    }
+
+    if program_path.is_empty() {
+        program_path = project_dir
+            .join("examples/fibonacci.json")
+            .display()
+            .to_string();
+    }
+
+    println!("Loading program from: {}", program_path);
+
+    // Try to load as a PIE first, if that fails, try as a regular program
+    let start_load = Instant::now();
+    let program_input_contents = if program_path.ends_with(".zip") {
+        // Load as PIE - we just need to verify it loads
+        // TODO(idanh): Add an option to configure the hash function
+        let _cairo_pie = CairoPie::read_zip_file(&PathBuf::from(&program_path))?;
+        format!(
+            r#"{{
+                "tasks": [
+                  {{
+                    "type": "CairoPiePath",
+                    "path": "{}",
+                    "program_hash_function": "pedersen"
+                  }}
+                ],
+                "single_page": true
+            }}"#,
+            program_path
+        )
+    } else {
+        // Load as regular program - we just need to verify it loads
+        let _program = Program::from_file(PathBuf::from(&program_path).as_path(), Some("main"))?;
+        format!(
+            r#"{{
+                "tasks": [
+                  {{
+                    "path": "{}",
+                    "program_hash_function": "pedersen",
+                    "type": "RunProgramTask"
+                  }}
+                ],
+                "single_page": true
+            }}"#,
+            program_path
+        )
+    };
+    let load_duration = start_load.elapsed();
+    println!("Program loaded in: {:?}", load_duration);
+
+    let simple_bootloader_program =
+        Program::from_file(simple_bootloader_compiled_path.as_path(), Some("main"))?;
+
+    let cairo_run_config = RunMode::Proof {
+        layout: LayoutName::starknet_with_keccak,
+        dynamic_layout_params: None,
+        disable_trace_padding: false,
+    }
+    .create_config();
+
+    println!("\nExecuting program...");
+    let start_exec = Instant::now();
+    let mut runner = cairo_run_program(
+        &simple_bootloader_program,
+        Some(program_input_contents),
+        cairo_run_config,
+    )?;
+    let exec_duration = start_exec.elapsed();
+    println!("Program executed in: {:?}", exec_duration);
+
+    // Verify execution by checking outputs
+    let mut output_buffer = String::new();
+    runner.vm.write_output(&mut output_buffer)?;
+
+    // Print execution verification info
+    println!("\nProgram Execution Verification:");
+    println!("Output size: {} bytes", output_buffer.len());
+
+    if show_output {
+        println!("Program output:\n{}", output_buffer);
+    }
+
+    // Print execution resources
+    println!("\nExecution Resources:");
+    let resources = runner.get_execution_resources()?;
+    println!("n_steps: {}", resources.n_steps);
+    println!("n_memory_holes: {}", resources.n_memory_holes);
+    println!(
+        "builtin_instance_counter: {:#?}",
+        resources.builtin_instance_counter
+    );
+
+    // Check if output is empty - might indicate optimization skip
+    if output_buffer.trim().is_empty() {
+        println!("WARNING: Empty output might indicate optimization issues!");
+    }
+
+    println!("\nBenchmark Summary:");
+    println!("Program load time: {:?}", load_duration);
+    println!("Program execution time: {:?}", exec_duration);
+    println!("Total time: {:?}", load_duration + exec_duration);
+
+    Ok(())
+}