feat: add --evals flag to logs and audit commands to filter by evals results

pkg/cli/audit.go

@@ -35,6 +35,7 @@ type AuditOptions struct {
 	ArtifactSets     []string
 	ExperimentFilter string
 	VariantFilter    string
+	EvalsOnly        bool
 }
 
 var auditCommandLong = `Audit one or more workflow runs by downloading artifacts and logs, detecting errors,
@@ -82,6 +83,7 @@ type auditCommandOptions struct {
 	stdin            bool
 	experimentFilter string
 	variantFilter    string
+	evalsOnly        bool
 }
 
 // NewAuditCommand creates the audit command
@@ -109,6 +111,7 @@ func registerAuditCommandFlags(cmd *cobra.Command) {
 	cmd.Flags().Bool("stdin", false, "Read workflow run IDs or URLs from stdin (one per line) instead of positional arguments")
 	cmd.Flags().String("experiment", "", "Filter to runs that include this experiment name")
 	cmd.Flags().String("variant", "", "Filter to runs with a specific variant value (requires --experiment)")
+	cmd.Flags().Bool("evals", false, "Skip runs that do not contain evals results (evals.jsonl); automatically downloads the evals artifact when --artifacts is narrowed")
 	RegisterDirFlagCompletion(cmd, "output")
 }
 
@@ -124,6 +127,12 @@ func runAuditCommand(cmd *cobra.Command, args []string) error {
 	if len(args) == 1 {
 		return runAuditSingle(cmd.Context(), args[0], opts)
 	}
+	if opts.evalsOnly {
+		return errors.New(console.FormatErrorWithSuggestions(
+			"--evals is not supported in multi-run diff mode",
+			[]string{"Provide a single run ID with --evals to filter by evals results"},
+		))
+	}
 	return runAuditMulti(cmd.Context(), args, opts.repoFlag, opts.outputDir, opts.verbose, opts.jsonOutput, opts.format, opts.artifacts)
 }
 
@@ -139,12 +148,21 @@ func getAuditCommandOptions(cmd *cobra.Command) (auditCommandOptions, error) {
 	opts.stdin, _ = cmd.Flags().GetBool("stdin")
 	opts.experimentFilter, _ = cmd.Flags().GetString("experiment")
 	opts.variantFilter, _ = cmd.Flags().GetString("variant")
+	opts.evalsOnly, _ = cmd.Flags().GetBool("evals")
 	if opts.variantFilter != "" && opts.experimentFilter == "" {
 		return auditCommandOptions{}, errors.New(console.FormatErrorWithSuggestions(
 			"--variant requires --experiment to be specified",
 			[]string{"Add --experiment <name> to filter by experiment name alongside --variant"},
 		))
 	}
+	// Auto-include the evals artifact when --evals is specified and the user has
+	// narrowed the artifact set (non-empty --artifacts).  When --artifacts is empty
+	// the default is "all", which already includes every artifact including evals,
+	// so we must not append here: doing so would change the default from "all" to
+	// "evals-only" and omit the activation/agent artifacts required for a full report.
+	if len(opts.artifacts) > 0 {
+		opts.artifacts = applyEvalsArtifact(opts.artifacts, opts.evalsOnly)
+	}
 	return opts, nil
 }
 
@@ -199,6 +217,7 @@ func runAuditSingle(ctx context.Context, runIDOrURL string, opts auditCommandOpt
 		ArtifactSets:     opts.artifacts,
 		ExperimentFilter: opts.experimentFilter,
 		VariantFilter:    opts.variantFilter,
+		EvalsOnly:        opts.evalsOnly,
 	})
 }
 
@@ -302,6 +321,7 @@ type auditRunConfig struct {
 	artifactFilter   []string
 	experimentFilter string
 	variantFilter    string
+	evalsOnly        bool
 }
 
 type auditAnalysisResults struct {
@@ -355,6 +375,9 @@ func AuditWorkflowRun(ctx context.Context, runID int64, opts AuditOptions) error
 	if shouldSkipAuditRun(cfg.runID, cfg.outputDir, cfg.experimentFilter, cfg.variantFilter) {
 		return nil
 	}
+	if shouldSkipForEvals(cfg) {
+		return nil
+	}
 	return renderAuditReport(ctx, processedRun, results.metrics, results.mcpToolUsage, cfg.auditOptions())
 }
 
@@ -376,6 +399,7 @@ func newAuditRunConfig(runID int64, opts AuditOptions) (auditRunConfig, error) {
 		artifactFilter:   ResolveArtifactFilter(opts.ArtifactSets),
 		experimentFilter: opts.ExperimentFilter,
 		variantFilter:    opts.VariantFilter,
+		evalsOnly:        opts.EvalsOnly,
 	}, nil
 }
 
@@ -454,6 +478,7 @@ func (cfg auditRunConfig) auditOptions() AuditOptions {
 		Verbose:    cfg.verbose,
 		Parse:      cfg.parse,
 		JSONOutput: cfg.jsonOutput,
+		EvalsOnly:  cfg.evalsOnly,
 	}
 }
 
@@ -469,6 +494,14 @@ func renderCachedAuditIfAvailable(ctx context.Context, cfg auditRunConfig) (bool
 	if shouldSkipAuditRun(cfg.runID, cfg.outputDir, cfg.experimentFilter, cfg.variantFilter) {
 		return true, nil
 	}
+	// When --evals is set but the evals artifact is not present locally, the cache
+	// was created without evals (e.g., default usage-only download).  Bypass the
+	// cache so prepareAuditWorkflowRun can fetch the evals artifact; the filter at
+	// the post-download check will then correctly decide whether to skip the run.
+	if cfg.evalsOnly && !runHasEvals(cfg.outputDir, cfg.verbose) {
+		auditLog.Printf("Cache miss for run %d evals: evals artifact not present locally, bypassing cache", cfg.runID)
+		return false, nil
+	}
 	processedRun := processedRunFromSummary(summary, cfg.outputDir)
 	return true, renderAuditReport(ctx, processedRun, summary.Metrics, summary.MCPToolUsage, cfg.auditOptions())
 }
@@ -931,3 +964,20 @@ func renderAuditCompletion(runOutputDir string, jsonOutput bool) {
 	fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Audit complete. Logs saved to "+absOutputDir))
 	fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Tip: use --artifacts to select specific artifact sets (agent, firewall, mcp, activation, detection, etc.)"))
 }
+
+// shouldSkipForEvals returns true when evals filtering is active but no evals results
+// are found locally after download. It logs the skip decision and, when verbose, prints
+// an info message to stderr. Call this only after artifact download has completed.
+func shouldSkipForEvals(cfg auditRunConfig) bool {
+	if !cfg.evalsOnly {
+		return false
+	}
+	if runHasEvals(cfg.outputDir, cfg.verbose) {
+		return false
+	}
+	auditLog.Printf("Skipping run %d: no evals results found (filtered by --evals)", cfg.runID)
+	if cfg.verbose {
+		fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow does not have evals results (filtered by --evals)", cfg.runID)))
+	}
+	return true
+}

pkg/cli/logs_artifact_set.go

@@ -68,6 +68,10 @@ const (
 	// ArtifactSetUsage downloads the compact usage artifact produced by the
 	// conclusion job (aw-info.jsonl, usage summaries, token usage JSONL).
 	ArtifactSetUsage ArtifactSet = "usage"
+
+	// ArtifactSetEvals downloads the evals artifact containing BinEval evaluation
+	// results (evals.jsonl) produced by the evals job.
+	ArtifactSetEvals ArtifactSet = "evals"
 )
 
 // artifactSetArtifacts maps each named set to the list of artifact base names it includes.
@@ -87,6 +91,8 @@ var artifactSetArtifacts = map[ArtifactSet][]string{
 	ArtifactSetExperiment: {constants.ExperimentArtifactName},
 	// usage: compact conclusion artifact for lightweight reporting/forecasting.
 	ArtifactSetUsage: {constants.UsageArtifactName},
+	// evals: BinEval evaluation results uploaded by the evals job.
+	ArtifactSetEvals: {constants.EvalsArtifactName},
 }
 
 const maxArtifactHintExamples = 2
@@ -276,3 +282,19 @@ func findMissingFilterEntries(filter []string, outputDir string) []string {
 	}
 	return missing
 }
+
+// applyEvalsArtifact appends the evals artifact set to artifacts when evalsOnly is true
+// and neither ArtifactSetEvals nor ArtifactSetAll is already present. This ensures
+// evals.jsonl is downloaded without requiring the user to also pass --artifacts evals.
+//
+// Note: callers that treat an empty artifacts slice as "all" (e.g., the audit command)
+// should guard with len(artifacts) > 0 before calling this function, to avoid
+// changing the empty/"all" default into an evals-only download.
+func applyEvalsArtifact(artifacts []string, evalsOnly bool) []string {
+	if evalsOnly &&
+		!slices.Contains(artifacts, string(ArtifactSetEvals)) &&
+		!slices.Contains(artifacts, string(ArtifactSetAll)) {
+		return append(artifacts, string(ArtifactSetEvals))
+	}
+	return artifacts
+}

pkg/cli/logs_artifact_set_test.go

@@ -252,7 +252,7 @@ func TestValidArtifactSetNames(t *testing.T) {
 	names := ValidArtifactSetNames()
 	require.NotEmpty(t, names, "ValidArtifactSetNames should return non-empty slice")
 
-	expected := []string{"all", "activation", "agent", "detection", "experiment", "firewall", "github-api", "mcp", "usage"}
+	expected := []string{"all", "activation", "agent", "detection", "evals", "experiment", "firewall", "github-api", "mcp", "usage"}
 	assert.ElementsMatch(t, expected, names, "ValidArtifactSetNames should contain all known sets")
 }
 

pkg/cli/logs_command.go

@@ -83,6 +83,7 @@ Downloaded artifacts include (when using --artifacts all):
   ` + string(constants.CLIExtensionPrefix) + ` logs --ref main                # Filter logs by branch or tag
   ` + string(constants.CLIExtensionPrefix) + ` logs --ref feature-xyz         # Filter logs by feature branch
   ` + string(constants.CLIExtensionPrefix) + ` logs --filtered-integrity      # Filter logs containing items that were filtered by gateway integrity checks
+  ` + string(constants.CLIExtensionPrefix) + ` logs --evals                    # Filter logs from workflows with evals results
   ` + string(constants.CLIExtensionPrefix) + ` logs --exclude-staged          # Exclude staged workflow runs from results
 
   # Run ID range filtering
@@ -152,6 +153,7 @@ Downloaded artifacts include (when using --artifacts all):
 				summaryFile, _ := cmd.Flags().GetString("summary-file")
 				safeOutputType, _ := cmd.Flags().GetString("safe-output")
 				filteredIntegrity, _ := cmd.Flags().GetBool("filtered-integrity")
+				evalsOnly, _ := cmd.Flags().GetBool("evals")
 				train, _ := cmd.Flags().GetBool("train")
 				format, _ := cmd.Flags().GetString("format")
 				reportFile, _ := cmd.Flags().GetString("report-file")
@@ -170,6 +172,8 @@ Downloaded artifacts include (when using --artifacts all):
 					return err
 				}
 
+				artifacts = applyEvalsArtifact(artifacts, evalsOnly)
+
 				return DownloadWorkflowLogsFromStdin(cmd.Context(), StdinLogsOptions{
 					RunURLs:           runURLs,
 					OutputDir:         outputDir,
@@ -186,6 +190,7 @@ Downloaded artifacts include (when using --artifacts all):
 					SummaryFile:       summaryFile,
 					SafeOutputType:    safeOutputType,
 					FilteredIntegrity: filteredIntegrity,
+					EvalsOnly:         evalsOnly,
 					Train:             train,
 					Format:            format,
 					ReportFile:        reportFile,
@@ -273,6 +278,7 @@ Downloaded artifacts include (when using --artifacts all):
 			summaryFile, _ := cmd.Flags().GetString("summary-file")
 			safeOutputType, _ := cmd.Flags().GetString("safe-output")
 			filteredIntegrity, _ := cmd.Flags().GetBool("filtered-integrity")
+			evalsOnly, _ := cmd.Flags().GetBool("evals")
 			train, _ := cmd.Flags().GetBool("train")
 			format, _ := cmd.Flags().GetString("format")
 			reportFile, _ := cmd.Flags().GetString("report-file")
@@ -321,6 +327,8 @@ Downloaded artifacts include (when using --artifacts all):
 
 			logsCommandLog.Printf("Executing logs download: workflow=%s, count=%d, engine=%s, train=%v, cache_before=%s", workflowName, count, engine, train, cacheBefore)
 
+			artifacts = applyEvalsArtifact(artifacts, evalsOnly)
+
 			return DownloadWorkflowLogs(cmd.Context(), LogsDownloadOptions{
 				WorkflowName:      workflowName,
 				Count:             count,
@@ -343,6 +351,7 @@ Downloaded artifacts include (when using --artifacts all):
 				SummaryFile:       summaryFile,
 				SafeOutputType:    safeOutputType,
 				FilteredIntegrity: filteredIntegrity,
+				EvalsOnly:         evalsOnly,
 				Train:             train,
 				Format:            format,
 				ReportFile:        reportFile,
@@ -368,6 +377,7 @@ Downloaded artifacts include (when using --artifacts all):
 	logsCmd.Flags().Bool("no-firewall", false, "Filter to only runs without firewall enabled")
 	logsCmd.Flags().String("safe-output", "", "Filter to runs containing a specific safe output type (e.g., create-issue, missing-tool, missing-data, noop, report-incomplete)")
 	logsCmd.Flags().Bool("filtered-integrity", false, "Filter to runs containing items that were filtered by gateway integrity checks")
+	logsCmd.Flags().Bool("evals", false, "Filter to runs containing evals results (evals.jsonl); automatically includes the evals artifact")
 	logsCmd.Flags().Bool("parse", false, "Run JavaScript parsers on agent logs and firewall logs, writing Markdown to log.md and firewall.md")
 	addJSONFlag(logsCmd)
 	logsCmd.Flags().Int("timeout", 0, "Download timeout in minutes (0 = no timeout)")

pkg/cli/logs_orchestrator.go

@@ -168,12 +168,13 @@ func DownloadWorkflowLogs(ctx context.Context, opts LogsDownloadOptions) error {
 	summaryFile := opts.SummaryFile
 	safeOutputType := opts.SafeOutputType
 	filteredIntegrity := opts.FilteredIntegrity
+	evalsOnly := opts.EvalsOnly
 	train := opts.Train
 	format := opts.Format
 	artifactSets := opts.ArtifactSets
 	after := opts.After
 
-	logsOrchestratorLog.Printf("Starting workflow log download: workflow=%s, count=%d, startDate=%s, endDate=%s, outputDir=%s, summaryFile=%s, safeOutputType=%s, filteredIntegrity=%v, train=%v, format=%s, artifactSets=%v, after=%s", workflowName, count, startDate, endDate, outputDir, summaryFile, safeOutputType, filteredIntegrity, train, format, artifactSets, after)
+	logsOrchestratorLog.Printf("Starting workflow log download: workflow=%s, count=%d, startDate=%s, endDate=%s, outputDir=%s, summaryFile=%s, safeOutputType=%s, filteredIntegrity=%v, evalsOnly=%v, train=%v, format=%s, artifactSets=%v, after=%s", workflowName, count, startDate, endDate, outputDir, summaryFile, safeOutputType, filteredIntegrity, evalsOnly, train, format, artifactSets, after)
 
 	// Validate and resolve artifact sets into a concrete filter (list of artifact base names).
 	if err := ValidateArtifactSets(artifactSets); err != nil {
@@ -272,6 +273,7 @@ func DownloadWorkflowLogs(ctx context.Context, opts LogsDownloadOptions) error {
 		noFirewall:        noFirewall,
 		safeOutputType:    safeOutputType,
 		filteredIntegrity: filteredIntegrity,
+		evalsOnly:         evalsOnly,
 	}
 
 	// Iterative algorithm: keep fetching runs until we have enough or exhaust available runs

pkg/cli/logs_orchestrator_filters.go

@@ -22,6 +22,7 @@ type runFilterOpts struct {
 	noFirewall        bool
 	safeOutputType    string
 	filteredIntegrity bool
+	evalsOnly         bool
 }
 
 var fetchJobStatusesForProcessedRun = fetchJobStatuses
@@ -131,6 +132,17 @@ func applyRunFilters(result DownloadResult, opts runFilterOpts, verbose bool) bo
 		}
 	}
 
+	// Apply evals filtering if --evals flag is specified.
+	if opts.evalsOnly {
+		if !runHasEvals(result.LogsPath, verbose) {
+			logsOrchestratorLog.Printf("Skipping run %d: no evals results found, filtered by --evals", result.Run.DatabaseID)
+			if verbose {
+				fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow does not have evals results (filtered by --evals)", result.Run.DatabaseID)))
+			}
+			return true
+		}
+	}
+
 	return false
 }
 

pkg/cli/logs_orchestrator_stdin.go

@@ -156,6 +156,7 @@ func DownloadWorkflowLogsFromStdin(ctx context.Context, opts StdinLogsOptions) e
 		noFirewall:        opts.NoFirewall,
 		safeOutputType:    opts.SafeOutputType,
 		filteredIntegrity: opts.FilteredIntegrity,
+		evalsOnly:         opts.EvalsOnly,
 	}
 
 	// Process download results applying the same filters as DownloadWorkflowLogs.

pkg/cli/logs_orchestrator_types.go

@@ -27,6 +27,7 @@ type LogsDownloadOptions struct {
 	SummaryFile       string
 	SafeOutputType    string
 	FilteredIntegrity bool
+	EvalsOnly         bool
 	Train             bool
 	Format            string
 	ArtifactSets      []string
@@ -51,6 +52,7 @@ type StdinLogsOptions struct {
 	SummaryFile       string
 	SafeOutputType    string
 	FilteredIntegrity bool
+	EvalsOnly         bool
 	Train             bool
 	Format            string
 	ReportFile        string