fix(prof): openvm-prof handling of duplicate names (#1617)

Author: jonathanpwang

.github/workflows/benchmark-call.yml

@@ -182,9 +182,9 @@ jobs:
       - name: Set working directory
         id: set-working-dir
         run: |
-          WORKING_DIR=$(jq -r --arg name "${{ inputs.benchmark_name }}" '
+          WORKING_DIR=$(jq -r --arg id "${{ inputs.benchmark_id }}" '
             .benchmarks[] |
-            select(.name == $name) |
+            select(.id == $id) |
             .working_directory
           ' ./ci/benchmark-config.json)
           RELATIVE_PATH=$(python3 -c "import os.path; print(os.path.relpath('.', '$WORKING_DIR'))")

.github/workflows/benchmarks.yml

@@ -199,6 +199,9 @@ jobs:
       - name: Download all metric json files from S3
         run: |
           matrix=$(echo '${{ needs.create-matrix.outputs.matrix }}')
+          names=$(echo $matrix | jq -r '.[] | "\(.id)"')
+          names_list=$(echo -n "$names" | paste -sd "," -)
+
           json_files=$(echo $matrix | jq -r '
             .[] |
             "\(.id)-${{ env.CURRENT_SHA }}.json"')
@@ -224,6 +227,7 @@ jobs:
 
           openvm-prof --json-paths "${json_file_list}" \
             --prev-json-paths "${prev_json_file_list}" \
+            --names "${names_list}" \
             summary \
             --benchmark-results-link "https://github.com/${{ github.repository }}/blob/benchmark-results/${BENCHMARK_RESULTS_PATH}" \
             --summary-md-path summary.md

crates/prof/src/main.rs

@@ -29,6 +29,12 @@ struct Cli {
     #[arg(long, value_delimiter = ',')]
     prev_json_paths: Option<Vec<PathBuf>>,
 
+    /// Display names for each metrics file (optional).
+    /// If provided, must be same length and in same order as `json-paths`.
+    /// Otherwise, the app program name will be used.
+    #[arg(long, value_delimiter = ',')]
+    names: Option<Vec<String>>,
+
     /// Path to write the output JSON in BMF format
     #[arg(long)]
     output_json: Option<PathBuf>,
@@ -55,10 +61,21 @@ fn main() -> Result<()> {
     } else {
         vec![None; args.json_paths.len()]
     };
+    let names = args.names.unwrap_or_default();
+    let mut names = if names.len() == args.json_paths.len() {
+        names
+    } else {
+        vec!["".to_string(); args.json_paths.len()]
+    };
     let mut aggregated_metrics = Vec::new();
     let mut md_paths = Vec::new();
     let mut output = BenchmarkOutput::default();
-    for (metrics_path, prev_metrics_path) in args.json_paths.into_iter().zip_eq(prev_json_paths) {
+    for ((metrics_path, prev_metrics_path), name) in args
+        .json_paths
+        .into_iter()
+        .zip_eq(prev_json_paths)
+        .zip_eq(&mut names)
+    {
         let db = MetricDb::new(&metrics_path)?;
         let grouped = GroupedMetrics::new(&db, "group")?;
         let mut aggregated = grouped.aggregate();
@@ -71,8 +88,10 @@ fn main() -> Result<()> {
                 aggregated.set_diff(prev_aggregated.as_ref().unwrap());
             }
         }
-
-        output.insert(&aggregated.name(), aggregated.to_bencher_metrics());
+        if name.is_empty() {
+            *name = aggregated.name();
+        }
+        output.insert(name, aggregated.to_bencher_metrics());
         let mut writer = Vec::new();
         aggregated.write_markdown(&mut writer, VM_METRIC_NAMES)?;
 
@@ -95,8 +114,12 @@ fn main() -> Result<()> {
     if let Some(command) = args.command {
         match command {
             Commands::Summary(cmd) => {
-                let summary =
-                    GithubSummary::new(&aggregated_metrics, &md_paths, &cmd.benchmark_results_link);
+                let summary = GithubSummary::new(
+                    &names,
+                    &aggregated_metrics,
+                    &md_paths,
+                    &cmd.benchmark_results_link,
+                );
                 let mut writer = Vec::new();
                 summary.write_markdown(&mut writer)?;
                 if let Some(path) = cmd.summary_md_path {

crates/prof/src/summary.rs

@@ -42,14 +42,16 @@ pub struct SingleSummaryMetrics {
 
 impl GithubSummary {
     pub fn new(
+        names: &[String],
         aggregated_metrics: &[(AggregateMetrics, Option<AggregateMetrics>)],
         md_paths: &[PathBuf],
         benchmark_results_link: &str,
     ) -> Self {
         let rows = aggregated_metrics
             .iter()
             .zip_eq(md_paths.iter())
-            .map(|((aggregated, prev_aggregated), md_path)| {
+            .zip_eq(names)
+            .map(|(((aggregated, prev_aggregated), md_path), name)| {
                 let md_filename = md_path.file_name().unwrap().to_str().unwrap();
                 let mut row = aggregated.get_summary_row(md_filename).unwrap();
                 if let Some(prev_aggregated) = prev_aggregated {
@@ -60,6 +62,7 @@ impl GithubSummary {
                         }
                     }
                 }
+                row.name = name.clone();
                 row
             })
             .collect();