added basic benchmarker for the CairoProgramRunner running simple bootloader on cairo pies + script for deploying it

on many pies.

Author: Idanh-starkware

Cargo.toml

@@ -35,3 +35,12 @@ tempfile = "3.10.1"
 thiserror = "1.0.61"
 thiserror-no-std = "2.0.2"
 tracing = "0.1.40"
+
+[profile.release]
+opt-level = 3
+lto = true
+codegen-units = 1
+panic = "abort"
+
+[profile.dev]
+opt-level = 3

crates/cairo-program-runner-lib/Cargo.toml

@@ -3,6 +3,10 @@ name = "cairo-program-runner-lib"
 version = "0.1.0"
 edition = "2021"
 
+[[bin]]
+name = "benchmark_runner"
+path = "scripts/benchmark_runner.rs"
+
 
 [dependencies]
 bincode.workspace = true

crates/cairo-program-runner-lib/scripts/benchmark_runner.rs

@@ -0,0 +1,161 @@
+//! Benchmark Runner for Cairo PIE Files
+//!
+//! This program measures the performance of loading and executing Cairo PIEs
+//! wrapped in simple bootloader and ran by the Cairo Program Runner.
+//!
+//! Usage:
+//!   cargo run --bin benchmark_runner <path_to_pie_file> [--show-output]
+//!
+//! Arguments:
+//!   <path_to_pie_file>  Path to the Cairo PIE file (.zip) to benchmark
+//!   --show-output       Optional flag to display program output
+//!
+//! Output:
+//!   - Program load time
+//!   - Program execution time
+//!   - Total runtime
+//!   - Execution resources (steps, memory holes, builtin counters)
+//!
+//! This binary is typically run through the run_all_pies.sh script, which provides
+//! additional features:
+//!   - Running multiple PIE files
+//!   - Testing different Rust optimization levels
+//!   - Aggregating results into a single file
+//!
+//! Example:
+//!   Direct usage:
+//!     cargo run --bin benchmark_runner path/to/program.zip
+//!   Via script:
+//!     ./scripts/run_all_pies.sh --opt-levels=3 --pie-dir=path/to/pies
+
+use std::error::Error;
+use std::path::PathBuf;
+use std::time::Instant;
+
+use cairo_program_runner_lib::cairo_run_program;
+use cairo_program_runner_lib::types::RunMode;
+use cairo_vm::types::layout_name::LayoutName;
+use cairo_vm::types::program::Program;
+use cairo_vm::vm::runners::cairo_pie::CairoPie;
+
+fn main() -> Result<(), Box<dyn Error>> {
+    let project_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    let simple_bootloader_compiled_path =
+        project_dir.join("resources/compiled_programs/bootloaders/simple_bootloader_compiled.json");
+
+    // Parse command line args
+    let args: Vec<String> = std::env::args().collect();
+    let mut program_path = String::new();
+    let mut show_output = false;
+
+    // Simple arg parsing
+    for (i, arg) in args.iter().enumerate() {
+        if arg == "--show-output" {
+            show_output = true;
+        } else if i > 0 && !arg.starts_with("--") {
+            program_path = arg.clone();
+        }
+    }
+
+    if program_path.is_empty() {
+        program_path = project_dir
+            .join("examples/fibonacci.json")
+            .display()
+            .to_string();
+    }
+
+    println!("Loading program from: {}", program_path);
+
+    // Try to load as a PIE first, if that fails, try as a regular program
+    let start_load = Instant::now();
+    let program_input_contents = if program_path.ends_with(".zip") {
+        // Load as PIE - we just need to verify it loads
+        // TODO(idanh): Add an option to configure the hash function
+        let _cairo_pie = CairoPie::read_zip_file(&PathBuf::from(&program_path))?;
+        format!(
+            r#"{{
+                "tasks": [
+                  {{
+                    "type": "CairoPiePath",
+                    "path": "{}",
+                    "program_hash_function": "pedersen"
+                  }}
+                ],
+                "single_page": true
+            }}"#,
+            program_path
+        )
+    } else {
+        // Load as regular program - we just need to verify it loads
+        let _program = Program::from_file(PathBuf::from(&program_path).as_path(), Some("main"))?;
+        format!(
+            r#"{{
+                "tasks": [
+                  {{
+                    "path": "{}",
+                    "program_hash_function": "pedersen",
+                    "type": "RunProgramTask"
+                  }}
+                ],
+                "single_page": true
+            }}"#,
+            program_path
+        )
+    };
+    let load_duration = start_load.elapsed();
+    println!("Program loaded in: {:?}", load_duration);
+
+    let simple_bootloader_program =
+        Program::from_file(simple_bootloader_compiled_path.as_path(), Some("main"))?;
+
+    let cairo_run_config = RunMode::Proof {
+        layout: LayoutName::starknet_with_keccak,
+        dynamic_layout_params: None,
+        disable_trace_padding: false,
+    }
+    .create_config();
+
+    println!("\nExecuting program...");
+    let start_exec = Instant::now();
+    let mut runner = cairo_run_program(
+        &simple_bootloader_program,
+        Some(program_input_contents),
+        cairo_run_config,
+    )?;
+    let exec_duration = start_exec.elapsed();
+    println!("Program executed in: {:?}", exec_duration);
+
+    // Verify execution by checking outputs
+    let mut output_buffer = String::new();
+    runner.vm.write_output(&mut output_buffer)?;
+
+    // Print execution verification info
+    println!("\nProgram Execution Verification:");
+    println!("Output size: {} bytes", output_buffer.len());
+
+    if show_output {
+        println!("Program output:\n{}", output_buffer);
+    }
+
+    // Print execution resources
+    println!("\nExecution Resources:");
+    let resources = runner.get_execution_resources()?;
+    println!("n_steps: {}", resources.n_steps);
+    println!("n_memory_holes: {}", resources.n_memory_holes);
+    println!(
+        "builtin_instance_counter: {:#?}",
+        resources.builtin_instance_counter
+    );
+
+    // Check if output is empty - might indicate optimization skip
+    if output_buffer.trim().is_empty() {
+        println!("WARNING: Empty output might indicate optimization issues!");
+    }
+
+    println!("\nBenchmark Summary:");
+    println!("Program load time: {:?}", load_duration);
+    println!("Program execution time: {:?}", exec_duration);
+    println!("Total time: {:?}", load_duration + exec_duration);
+
+    Ok(())
+}

crates/cairo-program-runner-lib/scripts/pies/POTC-PIE.zip

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+# Script Description:
+# This script runs benchmarks of the program runner on the simple bootloader wrapping
+# Cairo PIE files with different Rust optimization levels.
+# It measures program load time, execution time, and execution resources of the run.
+# Results are saved to a file.
+#
+# Usage:
+#   ./run_all_pies.sh [--opt-levels=<levels>] [--pie-dir=path/to/pies]
+#
+# Options:
+#   --opt-levels=<levels>  Specify which Rust optimization levels to run (0-3)
+#                         Examples: --opt-levels=3 (default, runs only level 3)
+#                                  --opt-levels=23 (runs levels 2 and 3)
+#                                  --opt-levels=013 (runs levels 0, 1, and 3)
+#   --pie-dir=<path>      Path to directory containing PIE files
+#                         Default: ./pies
+#
+# Output:
+#   - Shows compilation and execution progress in the terminal
+#   - Saves detailed benchmark results to benchmark_results.txt in the scripts directory
+#   - For each PIE file and optimization level, captures:
+#     * Program load and execution times
+#     * Number of steps and memory holes
+#     * Detailed builtin instance counter information
+#
+# Examples:
+#   ./run_all_pies.sh                          # Run with default settings (opt-level 3)
+#   ./run_all_pies.sh --opt-levels=012        # Run with optimization levels 0, 1, and 2
+#   ./run_all_pies.sh --pie-dir=/path/to/pies # Use PIE files from custom directory
+
+# Default values
+SCRIPT_DIR="$(dirname "$0")"
+PIES_DIR="$SCRIPT_DIR/pies"
+RESULTS_FILE="$SCRIPT_DIR/benchmark_results.txt"
+OPT_LEVELS="3" # Default to only opt level 3
+
+# Parse command line arguments
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --opt-levels=*)
+            OPT_LEVELS="${1#*=}"
+            ;;
+        --pie-dir=*)
+            PIES_DIR="${1#*=}"
+            ;;
+        *)
+            echo "Unknown parameter: $1"
+            echo "Usage: $0 [--opt-levels=<levels>] [--pie-dir=path/to/pies]"
+            echo "  --opt-levels: Specify which optimization levels to run (0-3)"
+            echo "               Examples: --opt-levels=23 (runs levels 2 and 3)"
+            echo "                        --opt-levels=013 (runs levels 0, 1, and 3)"
+            echo "  --pie-dir:   Path to directory containing PIE files"
+            exit 1
+            ;;
+    esac
+    shift
+done
+
+# Clear previous results
+echo "Cairo PIE Benchmark Results" > "$RESULTS_FILE"
+date >> "$RESULTS_FILE"
+echo "----------------------------------------" >> "$RESULTS_FILE"
+
+# Function to run cargo with specific opt level
+run_with_opt_level() {
+    local pie_file="$1"
+    local opt_level="$2"
+    
+    echo "Running ${pie_file} with optimization level ${opt_level}"
+    echo "Running ${pie_file} with optimization level ${opt_level}" >> "$RESULTS_FILE"
+    
+    # Run cargo and show compilation/execution stages, but redirect only benchmark results and resources to file
+    if ! RUSTFLAGS="-C opt-level=${opt_level}" cargo run --bin benchmark_runner "${pie_file}" \
+        2> >(grep -v "future version of Rust\|future-incompat-report" >&2) | \
+        tee >(awk '!/Program (load|execution) time:/ && (/Program (loaded|executed)/ || /Execution Resources:/ || /^n_steps:/ || /^n_memory_holes:/ || /^builtin_instance_counter:/ || /^[[:space:]]*[a-z_]+:/ || /^}/ || /Total time:/)' >> "$RESULTS_FILE"); then
+        echo "Error running benchmark" >> "$RESULTS_FILE"
+    fi
+    echo "----------------------------------------" >> "$RESULTS_FILE"
+}
+
+# Find all .zip files in pies directory
+for pie_file in "$PIES_DIR"/*.zip; do
+    if [ -f "$pie_file" ]; then
+        echo "=== Testing $(basename "$pie_file") ==="
+        echo
+        
+        # Run with specified optimization levels
+        # Validate opt levels
+        for opt_level in $(echo "$OPT_LEVELS" | grep -o .); do
+            if [[ ! "$opt_level" =~ ^[0-3]$ ]]; then
+                echo "Error: Invalid optimization level '$opt_level'. Must be one of: 0, 1, 2, 3"
+                exit 1
+            fi
+            run_with_opt_level "$pie_file" "$opt_level"
+            echo
+        done
+        
+        echo "================================================"
+        echo
+    fi
+done