From 64c9b81e49e1f6d2610dc9978e216e391aa8546d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 04:08:13 +0000 Subject: [PATCH 1/4] feat: add --evals flag to logs and audit commands to filter workflows with evals results Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/audit.go | 34 ++++++++++++++++++++++++++++ pkg/cli/logs_artifact_set.go | 6 +++++ pkg/cli/logs_artifact_set_test.go | 2 +- pkg/cli/logs_command.go | 17 ++++++++++++++ pkg/cli/logs_orchestrator.go | 4 +++- pkg/cli/logs_orchestrator_filters.go | 12 ++++++++++ pkg/cli/logs_orchestrator_stdin.go | 1 + pkg/cli/logs_orchestrator_types.go | 2 ++ pkg/cli/logs_run_processor.go | 33 +++++++++++++++++++++++++++ 9 files changed, 109 insertions(+), 2 deletions(-) diff --git a/pkg/cli/audit.go b/pkg/cli/audit.go index fb25f9c4a8d..a2b70a37f2f 100644 --- a/pkg/cli/audit.go +++ b/pkg/cli/audit.go @@ -35,6 +35,7 @@ type AuditOptions struct { ArtifactSets []string ExperimentFilter string VariantFilter string + EvalsOnly bool } var auditCommandLong = `Audit one or more workflow runs by downloading artifacts and logs, detecting errors, @@ -82,6 +83,7 @@ type auditCommandOptions struct { stdin bool experimentFilter string variantFilter string + evalsOnly bool } // NewAuditCommand creates the audit command @@ -109,6 +111,7 @@ func registerAuditCommandFlags(cmd *cobra.Command) { cmd.Flags().Bool("stdin", false, "Read workflow run IDs or URLs from stdin (one per line) instead of positional arguments") cmd.Flags().String("experiment", "", "Filter to runs that include this experiment name") cmd.Flags().String("variant", "", "Filter to runs with a specific variant value (requires --experiment)") + cmd.Flags().Bool("evals", false, "Include evals results in audit report; automatically downloads the evals artifact") RegisterDirFlagCompletion(cmd, "output") } @@ -139,12 +142,29 @@ func getAuditCommandOptions(cmd *cobra.Command) (auditCommandOptions, error) { opts.stdin, _ = cmd.Flags().GetBool("stdin") opts.experimentFilter, _ = cmd.Flags().GetString("experiment") opts.variantFilter, _ = cmd.Flags().GetString("variant") + opts.evalsOnly, _ = cmd.Flags().GetBool("evals") if opts.variantFilter != "" && opts.experimentFilter == "" { return auditCommandOptions{}, errors.New(console.FormatErrorWithSuggestions( "--variant requires --experiment to be specified", []string{"Add --experiment to filter by experiment name alongside --variant"}, )) } + // Auto-include the evals artifact when --evals is specified. + if opts.evalsOnly && len(opts.artifacts) > 0 { + hasEvals := false + hasAll := false + for _, a := range opts.artifacts { + if a == string(ArtifactSetEvals) { + hasEvals = true + } + if a == string(ArtifactSetAll) { + hasAll = true + } + } + if !hasEvals && !hasAll { + opts.artifacts = append(opts.artifacts, string(ArtifactSetEvals)) + } + } return opts, nil } @@ -199,6 +219,7 @@ func runAuditSingle(ctx context.Context, runIDOrURL string, opts auditCommandOpt ArtifactSets: opts.artifacts, ExperimentFilter: opts.experimentFilter, VariantFilter: opts.variantFilter, + EvalsOnly: opts.evalsOnly, }) } @@ -302,6 +323,7 @@ type auditRunConfig struct { artifactFilter []string experimentFilter string variantFilter string + evalsOnly bool } type auditAnalysisResults struct { @@ -355,6 +377,11 @@ func AuditWorkflowRun(ctx context.Context, runID int64, opts AuditOptions) error if shouldSkipAuditRun(cfg.runID, cfg.outputDir, cfg.experimentFilter, cfg.variantFilter) { return nil } + if cfg.evalsOnly && !runHasEvals(cfg.outputDir, cfg.verbose) { + auditLog.Printf("Skipping run %d: no evals results found (filtered by --evals)", cfg.runID) + fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow does not have evals results (filtered by --evals)", cfg.runID))) + return nil + } return renderAuditReport(ctx, processedRun, results.metrics, results.mcpToolUsage, cfg.auditOptions()) } @@ -376,6 +403,7 @@ func newAuditRunConfig(runID int64, opts AuditOptions) (auditRunConfig, error) { artifactFilter: ResolveArtifactFilter(opts.ArtifactSets), experimentFilter: opts.ExperimentFilter, variantFilter: opts.VariantFilter, + evalsOnly: opts.EvalsOnly, }, nil } @@ -454,6 +482,7 @@ func (cfg auditRunConfig) auditOptions() AuditOptions { Verbose: cfg.verbose, Parse: cfg.parse, JSONOutput: cfg.jsonOutput, + EvalsOnly: cfg.evalsOnly, } } @@ -469,6 +498,11 @@ func renderCachedAuditIfAvailable(ctx context.Context, cfg auditRunConfig) (bool if shouldSkipAuditRun(cfg.runID, cfg.outputDir, cfg.experimentFilter, cfg.variantFilter) { return true, nil } + if cfg.evalsOnly && !runHasEvals(cfg.outputDir, cfg.verbose) { + auditLog.Printf("Skipping run %d: no evals results found (filtered by --evals)", cfg.runID) + fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow does not have evals results (filtered by --evals)", cfg.runID))) + return true, nil + } processedRun := processedRunFromSummary(summary, cfg.outputDir) return true, renderAuditReport(ctx, processedRun, summary.Metrics, summary.MCPToolUsage, cfg.auditOptions()) } diff --git a/pkg/cli/logs_artifact_set.go b/pkg/cli/logs_artifact_set.go index dad59c873e8..f6799af3043 100644 --- a/pkg/cli/logs_artifact_set.go +++ b/pkg/cli/logs_artifact_set.go @@ -68,6 +68,10 @@ const ( // ArtifactSetUsage downloads the compact usage artifact produced by the // conclusion job (aw-info.jsonl, usage summaries, token usage JSONL). ArtifactSetUsage ArtifactSet = "usage" + + // ArtifactSetEvals downloads the evals artifact containing BinEval evaluation + // results (evals.jsonl) produced by the evals job. + ArtifactSetEvals ArtifactSet = "evals" ) // artifactSetArtifacts maps each named set to the list of artifact base names it includes. @@ -87,6 +91,8 @@ var artifactSetArtifacts = map[ArtifactSet][]string{ ArtifactSetExperiment: {constants.ExperimentArtifactName}, // usage: compact conclusion artifact for lightweight reporting/forecasting. ArtifactSetUsage: {constants.UsageArtifactName}, + // evals: BinEval evaluation results uploaded by the evals job. + ArtifactSetEvals: {constants.EvalsArtifactName}, } const maxArtifactHintExamples = 2 diff --git a/pkg/cli/logs_artifact_set_test.go b/pkg/cli/logs_artifact_set_test.go index 77e1b6ac708..5da73f36c76 100644 --- a/pkg/cli/logs_artifact_set_test.go +++ b/pkg/cli/logs_artifact_set_test.go @@ -252,7 +252,7 @@ func TestValidArtifactSetNames(t *testing.T) { names := ValidArtifactSetNames() require.NotEmpty(t, names, "ValidArtifactSetNames should return non-empty slice") - expected := []string{"all", "activation", "agent", "detection", "experiment", "firewall", "github-api", "mcp", "usage"} + expected := []string{"all", "activation", "agent", "detection", "evals", "experiment", "firewall", "github-api", "mcp", "usage"} assert.ElementsMatch(t, expected, names, "ValidArtifactSetNames should contain all known sets") } diff --git a/pkg/cli/logs_command.go b/pkg/cli/logs_command.go index aec65f8f6f2..d85a0cdc1c3 100644 --- a/pkg/cli/logs_command.go +++ b/pkg/cli/logs_command.go @@ -13,6 +13,7 @@ import ( "errors" "fmt" "os" + "slices" "strings" "time" @@ -83,6 +84,7 @@ Downloaded artifacts include (when using --artifacts all): ` + string(constants.CLIExtensionPrefix) + ` logs --ref main # Filter logs by branch or tag ` + string(constants.CLIExtensionPrefix) + ` logs --ref feature-xyz # Filter logs by feature branch ` + string(constants.CLIExtensionPrefix) + ` logs --filtered-integrity # Filter logs containing items that were filtered by gateway integrity checks + ` + string(constants.CLIExtensionPrefix) + ` logs --evals # Filter logs from workflows with evals results ` + string(constants.CLIExtensionPrefix) + ` logs --exclude-staged # Exclude staged workflow runs from results # Run ID range filtering @@ -152,6 +154,7 @@ Downloaded artifacts include (when using --artifacts all): summaryFile, _ := cmd.Flags().GetString("summary-file") safeOutputType, _ := cmd.Flags().GetString("safe-output") filteredIntegrity, _ := cmd.Flags().GetBool("filtered-integrity") + evalsOnly, _ := cmd.Flags().GetBool("evals") train, _ := cmd.Flags().GetBool("train") format, _ := cmd.Flags().GetString("format") reportFile, _ := cmd.Flags().GetString("report-file") @@ -170,6 +173,10 @@ Downloaded artifacts include (when using --artifacts all): return err } + if evalsOnly && !slices.Contains(artifacts, string(ArtifactSetEvals)) && !slices.Contains(artifacts, string(ArtifactSetAll)) { + artifacts = append(artifacts, string(ArtifactSetEvals)) + } + return DownloadWorkflowLogsFromStdin(cmd.Context(), StdinLogsOptions{ RunURLs: runURLs, OutputDir: outputDir, @@ -186,6 +193,7 @@ Downloaded artifacts include (when using --artifacts all): SummaryFile: summaryFile, SafeOutputType: safeOutputType, FilteredIntegrity: filteredIntegrity, + EvalsOnly: evalsOnly, Train: train, Format: format, ReportFile: reportFile, @@ -273,6 +281,7 @@ Downloaded artifacts include (when using --artifacts all): summaryFile, _ := cmd.Flags().GetString("summary-file") safeOutputType, _ := cmd.Flags().GetString("safe-output") filteredIntegrity, _ := cmd.Flags().GetBool("filtered-integrity") + evalsOnly, _ := cmd.Flags().GetBool("evals") train, _ := cmd.Flags().GetBool("train") format, _ := cmd.Flags().GetString("format") reportFile, _ := cmd.Flags().GetString("report-file") @@ -321,6 +330,12 @@ Downloaded artifacts include (when using --artifacts all): logsCommandLog.Printf("Executing logs download: workflow=%s, count=%d, engine=%s, train=%v, cache_before=%s", workflowName, count, engine, train, cacheBefore) + // Auto-include the evals artifact when --evals is specified so the filter + // can find evals.jsonl without requiring the user to also pass --artifacts evals. + if evalsOnly && !slices.Contains(artifacts, string(ArtifactSetEvals)) && !slices.Contains(artifacts, string(ArtifactSetAll)) { + artifacts = append(artifacts, string(ArtifactSetEvals)) + } + return DownloadWorkflowLogs(cmd.Context(), LogsDownloadOptions{ WorkflowName: workflowName, Count: count, @@ -343,6 +358,7 @@ Downloaded artifacts include (when using --artifacts all): SummaryFile: summaryFile, SafeOutputType: safeOutputType, FilteredIntegrity: filteredIntegrity, + EvalsOnly: evalsOnly, Train: train, Format: format, ReportFile: reportFile, @@ -368,6 +384,7 @@ Downloaded artifacts include (when using --artifacts all): logsCmd.Flags().Bool("no-firewall", false, "Filter to only runs without firewall enabled") logsCmd.Flags().String("safe-output", "", "Filter to runs containing a specific safe output type (e.g., create-issue, missing-tool, missing-data, noop, report-incomplete)") logsCmd.Flags().Bool("filtered-integrity", false, "Filter to runs containing items that were filtered by gateway integrity checks") + logsCmd.Flags().Bool("evals", false, "Filter to runs containing evals results (evals.jsonl); automatically includes the evals artifact") logsCmd.Flags().Bool("parse", false, "Run JavaScript parsers on agent logs and firewall logs, writing Markdown to log.md and firewall.md") addJSONFlag(logsCmd) logsCmd.Flags().Int("timeout", 0, "Download timeout in minutes (0 = no timeout)") diff --git a/pkg/cli/logs_orchestrator.go b/pkg/cli/logs_orchestrator.go index f6cafba3d5e..02f810b7197 100644 --- a/pkg/cli/logs_orchestrator.go +++ b/pkg/cli/logs_orchestrator.go @@ -168,12 +168,13 @@ func DownloadWorkflowLogs(ctx context.Context, opts LogsDownloadOptions) error { summaryFile := opts.SummaryFile safeOutputType := opts.SafeOutputType filteredIntegrity := opts.FilteredIntegrity + evalsOnly := opts.EvalsOnly train := opts.Train format := opts.Format artifactSets := opts.ArtifactSets after := opts.After - logsOrchestratorLog.Printf("Starting workflow log download: workflow=%s, count=%d, startDate=%s, endDate=%s, outputDir=%s, summaryFile=%s, safeOutputType=%s, filteredIntegrity=%v, train=%v, format=%s, artifactSets=%v, after=%s", workflowName, count, startDate, endDate, outputDir, summaryFile, safeOutputType, filteredIntegrity, train, format, artifactSets, after) + logsOrchestratorLog.Printf("Starting workflow log download: workflow=%s, count=%d, startDate=%s, endDate=%s, outputDir=%s, summaryFile=%s, safeOutputType=%s, filteredIntegrity=%v, evalsOnly=%v, train=%v, format=%s, artifactSets=%v, after=%s", workflowName, count, startDate, endDate, outputDir, summaryFile, safeOutputType, filteredIntegrity, evalsOnly, train, format, artifactSets, after) // Validate and resolve artifact sets into a concrete filter (list of artifact base names). if err := ValidateArtifactSets(artifactSets); err != nil { @@ -272,6 +273,7 @@ func DownloadWorkflowLogs(ctx context.Context, opts LogsDownloadOptions) error { noFirewall: noFirewall, safeOutputType: safeOutputType, filteredIntegrity: filteredIntegrity, + evalsOnly: evalsOnly, } // Iterative algorithm: keep fetching runs until we have enough or exhaust available runs diff --git a/pkg/cli/logs_orchestrator_filters.go b/pkg/cli/logs_orchestrator_filters.go index 231dbd38cf2..8598803706a 100644 --- a/pkg/cli/logs_orchestrator_filters.go +++ b/pkg/cli/logs_orchestrator_filters.go @@ -22,6 +22,7 @@ type runFilterOpts struct { noFirewall bool safeOutputType string filteredIntegrity bool + evalsOnly bool } var fetchJobStatusesForProcessedRun = fetchJobStatuses @@ -131,6 +132,17 @@ func applyRunFilters(result DownloadResult, opts runFilterOpts, verbose bool) bo } } + // Apply evals filtering if --evals flag is specified. + if opts.evalsOnly { + if !runHasEvals(result.LogsPath, verbose) { + logsOrchestratorLog.Printf("Skipping run %d: no evals results found, filtered by --evals", result.Run.DatabaseID) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow does not have evals results (filtered by --evals)", result.Run.DatabaseID))) + } + return true + } + } + return false } diff --git a/pkg/cli/logs_orchestrator_stdin.go b/pkg/cli/logs_orchestrator_stdin.go index 8eb5b030b61..a359374bb82 100644 --- a/pkg/cli/logs_orchestrator_stdin.go +++ b/pkg/cli/logs_orchestrator_stdin.go @@ -156,6 +156,7 @@ func DownloadWorkflowLogsFromStdin(ctx context.Context, opts StdinLogsOptions) e noFirewall: opts.NoFirewall, safeOutputType: opts.SafeOutputType, filteredIntegrity: opts.FilteredIntegrity, + evalsOnly: opts.EvalsOnly, } // Process download results applying the same filters as DownloadWorkflowLogs. diff --git a/pkg/cli/logs_orchestrator_types.go b/pkg/cli/logs_orchestrator_types.go index 2bcc8dcf60d..ffae4455506 100644 --- a/pkg/cli/logs_orchestrator_types.go +++ b/pkg/cli/logs_orchestrator_types.go @@ -27,6 +27,7 @@ type LogsDownloadOptions struct { SummaryFile string SafeOutputType string FilteredIntegrity bool + EvalsOnly bool Train bool Format string ArtifactSets []string @@ -51,6 +52,7 @@ type StdinLogsOptions struct { SummaryFile string SafeOutputType string FilteredIntegrity bool + EvalsOnly bool Train bool Format string ReportFile string diff --git a/pkg/cli/logs_run_processor.go b/pkg/cli/logs_run_processor.go index 6de34e72bdb..d55b0ef8580 100644 --- a/pkg/cli/logs_run_processor.go +++ b/pkg/cli/logs_run_processor.go @@ -584,3 +584,36 @@ func runHasDifcFilteredItems(runDir string, verbose bool) (bool, error) { return gatewayMetrics.TotalFiltered > 0, nil } + +// runHasEvals checks whether a run's output directory contains an evals results file +// (evals.jsonl). The evals artifact may be stored directly as "evals/evals.jsonl" or +// with a workflow_call hash prefix as "{hash}-evals/evals.jsonl". +func runHasEvals(runDir string, verbose bool) bool { + logsOrchestratorLog.Printf("Checking run for evals results: dir=%s", runDir) + + entries, err := os.ReadDir(runDir) + if err != nil { + return false + } + + for _, entry := range entries { + if !entry.IsDir() { + continue + } + name := entry.Name() + // Match exact "evals" or workflow_call prefixed "{hash}-evals". + if name != constants.EvalsArtifactName && !strings.HasSuffix(name, "-"+constants.EvalsArtifactName) { + continue + } + evalsFile := filepath.Join(runDir, name, constants.EvalsResultFilename) + if fileutil.FileExists(evalsFile) { + logsOrchestratorLog.Printf("Found evals results at: %s", evalsFile) + return true + } + } + + if verbose { + logsOrchestratorLog.Printf("No evals results found in: %s", runDir) + } + return false +} From ac429a82d4ef80886d7b328bec8d1dc36822da5a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 04:09:45 +0000 Subject: [PATCH 2/4] fix: use slices.Contains in audit --evals auto-include logic for consistency Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/audit.go | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/pkg/cli/audit.go b/pkg/cli/audit.go index a2b70a37f2f..97ca52d6321 100644 --- a/pkg/cli/audit.go +++ b/pkg/cli/audit.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "strings" "sync" "time" @@ -149,21 +150,10 @@ func getAuditCommandOptions(cmd *cobra.Command) (auditCommandOptions, error) { []string{"Add --experiment to filter by experiment name alongside --variant"}, )) } - // Auto-include the evals artifact when --evals is specified. - if opts.evalsOnly && len(opts.artifacts) > 0 { - hasEvals := false - hasAll := false - for _, a := range opts.artifacts { - if a == string(ArtifactSetEvals) { - hasEvals = true - } - if a == string(ArtifactSetAll) { - hasAll = true - } - } - if !hasEvals && !hasAll { - opts.artifacts = append(opts.artifacts, string(ArtifactSetEvals)) - } + // Auto-include the evals artifact when --evals is specified so the filter + // can find evals.jsonl without requiring the user to also pass --artifacts evals. + if opts.evalsOnly && !slices.Contains(opts.artifacts, string(ArtifactSetEvals)) && !slices.Contains(opts.artifacts, string(ArtifactSetAll)) { + opts.artifacts = append(opts.artifacts, string(ArtifactSetEvals)) } return opts, nil } From 04309d3b1cf405f7a34449cdcf36d33d27446d66 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 08:54:50 +0000 Subject: [PATCH 3/4] fix: address review feedback on --evals flag implementation - runHasEvals: also check root-level evals.jsonl (after flattenSingleFileArtifacts) - audit --evals artifact auto-include: only append evals when user narrowed artifacts - logs cache bypass: skip cache when evals requested but not present locally - audit cache bypass: bypass cache when --evals set but evals missing locally - reject --evals in multi-run diff mode with a clear error message - fix --evals help text to describe filtering behavior (not report content) Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/audit.go | 28 +++++++++--- pkg/cli/logs_run_processor.go | 85 ++++++++++++++++++++++------------- 2 files changed, 74 insertions(+), 39 deletions(-) diff --git a/pkg/cli/audit.go b/pkg/cli/audit.go index 97ca52d6321..be1c2625cd2 100644 --- a/pkg/cli/audit.go +++ b/pkg/cli/audit.go @@ -112,7 +112,7 @@ func registerAuditCommandFlags(cmd *cobra.Command) { cmd.Flags().Bool("stdin", false, "Read workflow run IDs or URLs from stdin (one per line) instead of positional arguments") cmd.Flags().String("experiment", "", "Filter to runs that include this experiment name") cmd.Flags().String("variant", "", "Filter to runs with a specific variant value (requires --experiment)") - cmd.Flags().Bool("evals", false, "Include evals results in audit report; automatically downloads the evals artifact") + cmd.Flags().Bool("evals", false, "Skip runs that do not contain evals results (evals.jsonl); automatically downloads the evals artifact when --artifacts is narrowed") RegisterDirFlagCompletion(cmd, "output") } @@ -128,6 +128,12 @@ func runAuditCommand(cmd *cobra.Command, args []string) error { if len(args) == 1 { return runAuditSingle(cmd.Context(), args[0], opts) } + if opts.evalsOnly { + return errors.New(console.FormatErrorWithSuggestions( + "--evals is not supported in multi-run diff mode", + []string{"Provide a single run ID with --evals to filter by evals results"}, + )) + } return runAuditMulti(cmd.Context(), args, opts.repoFlag, opts.outputDir, opts.verbose, opts.jsonOutput, opts.format, opts.artifacts) } @@ -150,9 +156,14 @@ func getAuditCommandOptions(cmd *cobra.Command) (auditCommandOptions, error) { []string{"Add --experiment to filter by experiment name alongside --variant"}, )) } - // Auto-include the evals artifact when --evals is specified so the filter - // can find evals.jsonl without requiring the user to also pass --artifacts evals. - if opts.evalsOnly && !slices.Contains(opts.artifacts, string(ArtifactSetEvals)) && !slices.Contains(opts.artifacts, string(ArtifactSetAll)) { + // Auto-include the evals artifact when --evals is specified and the user has + // narrowed the artifact set (non-empty --artifacts). When --artifacts is empty + // the default is "all", which already includes every artifact including evals, + // so we must not append here: doing so would change the default from "all" to + // "evals-only" and omit the activation/agent artifacts required for a full report. + if opts.evalsOnly && len(opts.artifacts) > 0 && + !slices.Contains(opts.artifacts, string(ArtifactSetEvals)) && + !slices.Contains(opts.artifacts, string(ArtifactSetAll)) { opts.artifacts = append(opts.artifacts, string(ArtifactSetEvals)) } return opts, nil @@ -488,10 +499,13 @@ func renderCachedAuditIfAvailable(ctx context.Context, cfg auditRunConfig) (bool if shouldSkipAuditRun(cfg.runID, cfg.outputDir, cfg.experimentFilter, cfg.variantFilter) { return true, nil } + // When --evals is set but the evals artifact is not present locally, the cache + // was created without evals (e.g., default usage-only download). Bypass the + // cache so prepareAuditWorkflowRun can fetch the evals artifact; the filter at + // the post-download check will then correctly decide whether to skip the run. if cfg.evalsOnly && !runHasEvals(cfg.outputDir, cfg.verbose) { - auditLog.Printf("Skipping run %d: no evals results found (filtered by --evals)", cfg.runID) - fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow does not have evals results (filtered by --evals)", cfg.runID))) - return true, nil + auditLog.Printf("Cache miss for run %d evals: evals artifact not present locally, bypassing cache", cfg.runID) + return false, nil } processedRun := processedRunFromSummary(summary, cfg.outputDir) return true, renderAuditReport(ctx, processedRun, summary.Metrics, summary.MCPToolUsage, cfg.auditOptions()) diff --git a/pkg/cli/logs_run_processor.go b/pkg/cli/logs_run_processor.go index d55b0ef8580..ad9643450ca 100644 --- a/pkg/cli/logs_run_processor.go +++ b/pkg/cli/logs_run_processor.go @@ -16,6 +16,7 @@ import ( "math" "os" "path/filepath" + "slices" "strings" "sync/atomic" "time" @@ -111,37 +112,47 @@ func downloadRunArtifactsConcurrent(ctx context.Context, runs []WorkflowRun, out // Try to load cached summary first if summary, ok := loadRunSummary(runOutputDir, verbose); ok { - logsOrchestratorLog.Printf("Cache hit for run %d, using cached summary", run.DatabaseID) - // Valid cached summary exists, use it directly - result := DownloadResult{ - Run: summary.Run, - Metrics: summary.Metrics, - AwContext: summary.AwContext, - TaskDomain: summary.TaskDomain, - BehaviorFingerprint: summary.BehaviorFingerprint, - AgenticAssessments: summary.AgenticAssessments, - AccessAnalysis: summary.AccessAnalysis, - FirewallAnalysis: summary.FirewallAnalysis, - RedactedDomainsAnalysis: summary.RedactedDomainsAnalysis, - MissingTools: summary.MissingTools, - MissingData: summary.MissingData, - Noops: summary.Noops, - MCPFailures: summary.MCPFailures, - MCPToolUsage: summary.MCPToolUsage, - TokenUsage: summary.TokenUsage, - GitHubRateLimitUsage: summary.GitHubRateLimitUsage, - JobDetails: summary.JobDetails, - LogsPath: runOutputDir, - Cached: true, // Mark as cached - } - // Re-apply the usage activity backfill to heal stale cache entries. - backfillCacheHitIfNeeded(&result, runOutputDir, verbose) - // Update progress counter - completed := completedCount.Add(1) - if progressBar != nil { - fmt.Fprintf(os.Stderr, "Processing runs: %s\r", progressBar.Update(completed)) + // When the caller requested the evals artifact and it is not + // present in the cached run directory, the cache was created + // without evals (e.g., an earlier default usage-only download). + // Bypass the cache so the fresh download below can fetch evals; + // the post-download filter will then decide whether to skip. + evalsRequested := slices.Contains(artifactFilter, constants.EvalsArtifactName) + if evalsRequested && !runHasEvals(runOutputDir, verbose) { + logsOrchestratorLog.Printf("Cache bypass for run %d: evals artifact requested but not present locally", run.DatabaseID) + } else { + logsOrchestratorLog.Printf("Cache hit for run %d, using cached summary", run.DatabaseID) + // Valid cached summary exists, use it directly + result := DownloadResult{ + Run: summary.Run, + Metrics: summary.Metrics, + AwContext: summary.AwContext, + TaskDomain: summary.TaskDomain, + BehaviorFingerprint: summary.BehaviorFingerprint, + AgenticAssessments: summary.AgenticAssessments, + AccessAnalysis: summary.AccessAnalysis, + FirewallAnalysis: summary.FirewallAnalysis, + RedactedDomainsAnalysis: summary.RedactedDomainsAnalysis, + MissingTools: summary.MissingTools, + MissingData: summary.MissingData, + Noops: summary.Noops, + MCPFailures: summary.MCPFailures, + MCPToolUsage: summary.MCPToolUsage, + TokenUsage: summary.TokenUsage, + GitHubRateLimitUsage: summary.GitHubRateLimitUsage, + JobDetails: summary.JobDetails, + LogsPath: runOutputDir, + Cached: true, // Mark as cached + } + // Re-apply the usage activity backfill to heal stale cache entries. + backfillCacheHitIfNeeded(&result, runOutputDir, verbose) + // Update progress counter + completed := completedCount.Add(1) + if progressBar != nil { + fmt.Fprintf(os.Stderr, "Processing runs: %s\r", progressBar.Update(completed)) + } + return result, nil } - return result, nil } // No cached summary or version mismatch - download and process. @@ -586,11 +597,21 @@ func runHasDifcFilteredItems(runDir string, verbose bool) (bool, error) { } // runHasEvals checks whether a run's output directory contains an evals results file -// (evals.jsonl). The evals artifact may be stored directly as "evals/evals.jsonl" or -// with a workflow_call hash prefix as "{hash}-evals/evals.jsonl". +// (evals.jsonl). It looks in three locations: +// 1. runDir/evals.jsonl — produced when flattenSingleFileArtifacts collapsed the +// one-file evals artifact from its directory directly to the run root. +// 2. runDir/evals/evals.jsonl — un-flattened artifact directory. +// 3. runDir/{hash}-evals/evals.jsonl — workflow_call hash-prefixed variant. func runHasEvals(runDir string, verbose bool) bool { logsOrchestratorLog.Printf("Checking run for evals results: dir=%s", runDir) + // Case 1: flattenSingleFileArtifacts moved the file directly to the run root. + rootEvalsFile := filepath.Join(runDir, constants.EvalsResultFilename) + if fileutil.FileExists(rootEvalsFile) { + logsOrchestratorLog.Printf("Found evals results at: %s", rootEvalsFile) + return true + } + entries, err := os.ReadDir(runDir) if err != nil { return false From 53a7bb73c054162b5b86680a488992666de1aba5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:24:57 +0000 Subject: [PATCH 4/4] refactor: address github-actions bot feedback on --evals implementation Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- pkg/cli/audit.go | 28 ++++++++--- pkg/cli/logs_artifact_set.go | 16 ++++++ pkg/cli/logs_command.go | 11 +--- pkg/cli/logs_run_processor_test.go | 81 ++++++++++++++++++++++++++++++ 4 files changed, 119 insertions(+), 17 deletions(-) create mode 100644 pkg/cli/logs_run_processor_test.go diff --git a/pkg/cli/audit.go b/pkg/cli/audit.go index be1c2625cd2..66b350a809e 100644 --- a/pkg/cli/audit.go +++ b/pkg/cli/audit.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "path/filepath" - "slices" "strings" "sync" "time" @@ -161,10 +160,8 @@ func getAuditCommandOptions(cmd *cobra.Command) (auditCommandOptions, error) { // the default is "all", which already includes every artifact including evals, // so we must not append here: doing so would change the default from "all" to // "evals-only" and omit the activation/agent artifacts required for a full report. - if opts.evalsOnly && len(opts.artifacts) > 0 && - !slices.Contains(opts.artifacts, string(ArtifactSetEvals)) && - !slices.Contains(opts.artifacts, string(ArtifactSetAll)) { - opts.artifacts = append(opts.artifacts, string(ArtifactSetEvals)) + if len(opts.artifacts) > 0 { + opts.artifacts = applyEvalsArtifact(opts.artifacts, opts.evalsOnly) } return opts, nil } @@ -378,9 +375,7 @@ func AuditWorkflowRun(ctx context.Context, runID int64, opts AuditOptions) error if shouldSkipAuditRun(cfg.runID, cfg.outputDir, cfg.experimentFilter, cfg.variantFilter) { return nil } - if cfg.evalsOnly && !runHasEvals(cfg.outputDir, cfg.verbose) { - auditLog.Printf("Skipping run %d: no evals results found (filtered by --evals)", cfg.runID) - fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow does not have evals results (filtered by --evals)", cfg.runID))) + if shouldSkipForEvals(cfg) { return nil } return renderAuditReport(ctx, processedRun, results.metrics, results.mcpToolUsage, cfg.auditOptions()) @@ -969,3 +964,20 @@ func renderAuditCompletion(runOutputDir string, jsonOutput bool) { fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Audit complete. Logs saved to "+absOutputDir)) fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Tip: use --artifacts to select specific artifact sets (agent, firewall, mcp, activation, detection, etc.)")) } + +// shouldSkipForEvals returns true when evals filtering is active but no evals results +// are found locally after download. It logs the skip decision and, when verbose, prints +// an info message to stderr. Call this only after artifact download has completed. +func shouldSkipForEvals(cfg auditRunConfig) bool { + if !cfg.evalsOnly { + return false + } + if runHasEvals(cfg.outputDir, cfg.verbose) { + return false + } + auditLog.Printf("Skipping run %d: no evals results found (filtered by --evals)", cfg.runID) + if cfg.verbose { + fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow does not have evals results (filtered by --evals)", cfg.runID))) + } + return true +} diff --git a/pkg/cli/logs_artifact_set.go b/pkg/cli/logs_artifact_set.go index f6799af3043..97300ddda5f 100644 --- a/pkg/cli/logs_artifact_set.go +++ b/pkg/cli/logs_artifact_set.go @@ -282,3 +282,19 @@ func findMissingFilterEntries(filter []string, outputDir string) []string { } return missing } + +// applyEvalsArtifact appends the evals artifact set to artifacts when evalsOnly is true +// and neither ArtifactSetEvals nor ArtifactSetAll is already present. This ensures +// evals.jsonl is downloaded without requiring the user to also pass --artifacts evals. +// +// Note: callers that treat an empty artifacts slice as "all" (e.g., the audit command) +// should guard with len(artifacts) > 0 before calling this function, to avoid +// changing the empty/"all" default into an evals-only download. +func applyEvalsArtifact(artifacts []string, evalsOnly bool) []string { + if evalsOnly && + !slices.Contains(artifacts, string(ArtifactSetEvals)) && + !slices.Contains(artifacts, string(ArtifactSetAll)) { + return append(artifacts, string(ArtifactSetEvals)) + } + return artifacts +} diff --git a/pkg/cli/logs_command.go b/pkg/cli/logs_command.go index d85a0cdc1c3..322cc6a2efa 100644 --- a/pkg/cli/logs_command.go +++ b/pkg/cli/logs_command.go @@ -13,7 +13,6 @@ import ( "errors" "fmt" "os" - "slices" "strings" "time" @@ -173,9 +172,7 @@ Downloaded artifacts include (when using --artifacts all): return err } - if evalsOnly && !slices.Contains(artifacts, string(ArtifactSetEvals)) && !slices.Contains(artifacts, string(ArtifactSetAll)) { - artifacts = append(artifacts, string(ArtifactSetEvals)) - } + artifacts = applyEvalsArtifact(artifacts, evalsOnly) return DownloadWorkflowLogsFromStdin(cmd.Context(), StdinLogsOptions{ RunURLs: runURLs, @@ -330,11 +327,7 @@ Downloaded artifacts include (when using --artifacts all): logsCommandLog.Printf("Executing logs download: workflow=%s, count=%d, engine=%s, train=%v, cache_before=%s", workflowName, count, engine, train, cacheBefore) - // Auto-include the evals artifact when --evals is specified so the filter - // can find evals.jsonl without requiring the user to also pass --artifacts evals. - if evalsOnly && !slices.Contains(artifacts, string(ArtifactSetEvals)) && !slices.Contains(artifacts, string(ArtifactSetAll)) { - artifacts = append(artifacts, string(ArtifactSetEvals)) - } + artifacts = applyEvalsArtifact(artifacts, evalsOnly) return DownloadWorkflowLogs(cmd.Context(), LogsDownloadOptions{ WorkflowName: workflowName, diff --git a/pkg/cli/logs_run_processor_test.go b/pkg/cli/logs_run_processor_test.go new file mode 100644 index 00000000000..89645c49c01 --- /dev/null +++ b/pkg/cli/logs_run_processor_test.go @@ -0,0 +1,81 @@ +//go:build !integration + +package cli + +import ( + "os" + "path/filepath" + "testing" + + "github.com/github/gh-aw/pkg/constants" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRunHasEvals(t *testing.T) { + tests := []struct { + name string + setup func(t *testing.T, dir string) + expected bool + }{ + { + name: "root-level evals.jsonl (flattenSingleFileArtifacts output)", + setup: func(t *testing.T, dir string) { + t.Helper() + require.NoError(t, os.WriteFile(filepath.Join(dir, constants.EvalsResultFilename), []byte("{}"), 0600)) + }, + expected: true, + }, + { + name: "evals/evals.jsonl (un-flattened artifact directory)", + setup: func(t *testing.T, dir string) { + t.Helper() + evalsDir := filepath.Join(dir, constants.EvalsArtifactName) + require.NoError(t, os.Mkdir(evalsDir, 0700)) + require.NoError(t, os.WriteFile(filepath.Join(evalsDir, constants.EvalsResultFilename), []byte("{}"), 0600)) + }, + expected: true, + }, + { + name: "hash-prefixed {hash}-evals/evals.jsonl (workflow_call variant)", + setup: func(t *testing.T, dir string) { + t.Helper() + evalsDir := filepath.Join(dir, "abc123-"+constants.EvalsArtifactName) + require.NoError(t, os.Mkdir(evalsDir, 0700)) + require.NoError(t, os.WriteFile(filepath.Join(evalsDir, constants.EvalsResultFilename), []byte("{}"), 0600)) + }, + expected: true, + }, + { + name: "evals/ directory exists but contains no evals.jsonl", + setup: func(t *testing.T, dir string) { + t.Helper() + evalsDir := filepath.Join(dir, constants.EvalsArtifactName) + require.NoError(t, os.Mkdir(evalsDir, 0700)) + require.NoError(t, os.WriteFile(filepath.Join(evalsDir, "other.txt"), []byte("data"), 0600)) + }, + expected: false, + }, + { + name: "empty directory", + setup: func(t *testing.T, dir string) {}, + expected: false, + }, + { + name: "non-existent directory", + setup: func(t *testing.T, dir string) { + t.Helper() + require.NoError(t, os.RemoveAll(dir)) + }, + expected: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + tc.setup(t, dir) + assert.Equal(t, tc.expected, runHasEvals(dir, false)) + }) + } +}