diff --git a/.github/rulesets/protect-main.json b/.github/rulesets/protect-main.json index ca20e34..7d67a7f 100644 --- a/.github/rulesets/protect-main.json +++ b/.github/rulesets/protect-main.json @@ -21,7 +21,7 @@ { "type": "pull_request", "parameters": { - "required_approving_review_count": 0, + "required_approving_review_count": 1, "dismiss_stale_reviews_on_push": false, "required_reviewers": [], "require_code_owner_review": false, diff --git a/AGENTS.md b/AGENTS.md index 93f9459..46a76ba 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,13 +50,22 @@ incorrectly", parse stderr — usage errors include `Usage:` text; check failure - `src/project.rs` — project discovery and source file walking - `src/scorecard.rs` — output formatting (text and JSON) - `src/types.rs` — CheckResult, CheckStatus, CheckGroup, CheckLayer +- `src/principles/registry.rs` — single source of truth linking spec requirements (P1–P7 MUSTs/SHOULDs/MAYs) to the + checks that verify them +- `src/principles/matrix.rs` — coverage-matrix generator + drift detector ## Adding a New Check 1. Create a file in the appropriate `src/checks/` subdirectory -2. Implement the `Check` trait: `id()`, `group()`, `layer()`, `applicable()`, `run()` +2. Implement the `Check` trait: `id()`, `group()`, `layer()`, `applicable()`, `run()`, and `covers()` if the check + verifies requirements in `src/principles/registry.rs` (return a `&'static [&'static str]` of requirement IDs) 3. Register in the layer's `mod.rs` (e.g., `all_rust_checks()`) 4. Add inline `#[cfg(test)]` tests +5. Regenerate the coverage matrix: `cargo run -- generate coverage-matrix` (produces `docs/coverage-matrix.md` + + `coverage/matrix.json`, both tracked in git) + +See `CLAUDE.md` §"Principle Registry" and §"`covers()` Declaration" for the registry conventions and drift-detector +behavior. ## Testing diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b9074a..6924732 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,28 @@ All notable changes to this project will be documented in this file. +## [0.1.2] - 2026-04-21 + +### Added + +- Add `p1-flag-existence` behavioral check — passes when `--help` advertises a non-interactive gate flag (`--no-interactive`, `--batch`, `--headless`, `-y`, `--yes`, `-p`, `--print`, `--no-input`, `--assume-yes`). Skips when the target already satisfies P1 via help-on-bare-invocation or stdin-primary. by @brettdavies in [#24](https://github.com/brettdavies/agentnative-cli/pull/24) +- Add `p1-env-hints` behavioral check — passes when `--help` exposes clap-style `[env: FOO]` bindings for flags. Emits medium confidence; the heuristic covers the canonical but not the only env-binding format. +- Add `p6-no-pager-behavioral` behavioral check — passes when `--no-pager` is advertised in `--help`. Skips when no pager signal (`less` / `more` / `$PAGER` / `--pager`) appears. Emits medium confidence. +- Add `confidence` field to every scorecard result (`high` / `medium` / `low`). Additive; v1.1 consumers feature-detect. +- Add `dual_layer` count to the coverage matrix summary so the headline prose surfaces how many covered requirements have verifiers in two layers. + +### Changed + +- Raise required approving review count on `main` branch from 0 to 1. by @brettdavies in [#24](https://github.com/brettdavies/agentnative-cli/pull/24) + +### Documentation + +- Document the \`covers()\` trait method and the coverage-matrix regeneration step in the \"Adding a New Check\" guide. by @brettdavies in [#23](https://github.com/brettdavies/agentnative-cli/pull/23) +- Refresh README sample output to match v0.1.1 dogfood behaviour. +- Regenerate `docs/coverage-matrix.md` + `coverage/matrix.json` to pick up the three new behavioral verifiers. by @brettdavies in [#24](https://github.com/brettdavies/agentnative-cli/pull/24) + +**Full Changelog**: [v0.1.1...v0.1.2](https://github.com/brettdavies/agentnative-cli/compare/v0.1.1...v0.1.2) + ## [0.1.1] - 2026-04-20 ### Added diff --git a/Cargo.lock b/Cargo.lock index df5f3b1..d90ea12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,7 +4,7 @@ version = 4 [[package]] name = "agentnative" -version = "0.1.1" +version = "0.1.2" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index eec59c9..60c96f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "agentnative" -version = "0.1.1" +version = "0.1.2" edition = "2024" description = "The agent-native CLI linter — check whether your CLI follows agent-readiness principles" license = "MIT OR Apache-2.0" diff --git a/README.md b/README.md index fc5ebe4..b527548 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ P6 — Composable Structure Code Quality [PASS] No .unwrap() in source (code-unwrap) -30 checks: 20 pass, 8 warn, 0 fail, 2 skip, 0 error +30 checks: 26 pass, 2 warn, 0 fail, 2 skip, 0 error ``` ## Three Check Layers @@ -158,10 +158,11 @@ Pre-generated scripts are also available in `completions/`. anc check . --output json ``` -Produces a scorecard with results and summary: +Produces a scorecard (`schema_version: "1.1"`) with results, summary, and coverage against the 7 principles: ```json { + "schema_version": "1.1", "results": [ { "id": "p3-help", @@ -174,15 +175,27 @@ Produces a scorecard with results and summary: ], "summary": { "total": 30, - "pass": 20, - "warn": 8, + "pass": 26, + "warn": 2, "fail": 0, "skip": 2, "error": 0 - } + }, + "coverage_summary": { + "must": { "total": 23, "verified": 17 }, + "should": { "total": 16, "verified": 2 }, + "may": { "total": 7, "verified": 0 } + }, + "audience": null, + "audit_profile": null } ``` +- `coverage_summary` — how many MUSTs/SHOULDs/MAYs the checks that ran actually verified, against the spec registry's + totals. See `docs/coverage-matrix.md` for the per-requirement breakdown. +- `audience` / `audit_profile` — reserved for v0.1.3 (audience classifier + `registry.yaml` suppression). Serialize as + `null` today; consumers should feature-detect. + ## Contributing ```bash @@ -192,6 +205,24 @@ cargo test cargo run -- check . ``` +### Reporting issues + +Open an issue at +[github.com/brettdavies/agentnative-cli/issues/new/choose](https://github.com/brettdavies/agentnative-cli/issues/new/choose). +Seven structured templates cover the common cases: + +| Template | Use it when | +| --- | --- | +| False positive | A check flagged your CLI but you believe your CLI is doing the right thing. | +| Scoring bug | Results don't match what the check should be doing (wrong status, miscategorized group/layer, evidence pointing at the wrong line). | +| Feature request | Missing capability, flag, or output format in the checker itself. | +| Grade a CLI | Nominate a CLI for an `anc`-graded readiness review. | +| Pressure test | Challenge a principle or check definition — "this check is too strict / too loose / wrong on this class of CLI." | +| Spec question | Ambiguity or gap in the 7-principle spec (not the checker). | +| Something else | Chooser for anything outside the templates above. | + +Filing on the right template front-loads the triage context we need and keeps issues out of a single-bucket backlog. + ## License MIT OR Apache-2.0 diff --git a/coverage/matrix.json b/coverage/matrix.json index ebd6263..13c5cee 100644 --- a/coverage/matrix.json +++ b/coverage/matrix.json @@ -11,6 +11,10 @@ "kind": "universal" }, "verifiers": [ + { + "check_id": "p1-env-hints", + "layer": "behavioral" + }, { "check_id": "p1-env-flags-source", "layer": "source" @@ -30,6 +34,10 @@ "check_id": "p1-non-interactive", "layer": "behavioral" }, + { + "check_id": "p1-flag-existence", + "layer": "behavioral" + }, { "check_id": "p1-non-interactive-source", "layer": "project" @@ -439,6 +447,10 @@ "condition": "CLI invokes a pager for output" }, "verifiers": [ + { + "check_id": "p6-no-pager-behavioral", + "layer": "behavioral" + }, { "check_id": "p6-no-pager", "layer": "source" @@ -602,6 +614,7 @@ "total": 46, "covered": 19, "uncovered": 27, + "dual_layer": 7, "must": { "total": 23, "covered": 17 diff --git a/docs/coverage-matrix.md b/docs/coverage-matrix.md index 98a54d0..157dc24 100644 --- a/docs/coverage-matrix.md +++ b/docs/coverage-matrix.md @@ -8,6 +8,7 @@ When a requirement has no verifier, the cell reads **UNCOVERED** and the reader ## Summary - **Total**: 46 requirements (19 covered / 27 uncovered) +- **Dual-layer**: 7 of 19 covered requirements have verifiers in two layers (behavioral + source or project) - **MUST**: 17 of 23 covered - **SHOULD**: 2 of 16 covered - **MAY**: 0 of 7 covered @@ -16,8 +17,8 @@ When a requirement has no verifier, the cell reads **UNCOVERED** and the reader | ID | Level | Applicability | Verifier(s) | Summary | | --- | --- | --- | --- | --- | -| `p1-must-env-var` | MUST | Universal | `p1-env-flags-source` (source) | Every flag settable via environment variable (falsey-value parser for booleans). | -| `p1-must-no-interactive` | MUST | Universal | `p1-non-interactive` (behavioral)
`p1-non-interactive-source` (project) | `--no-interactive` flag gates every prompt library call; when set or stdin is not a TTY, use defaults/stdin or exit with an actionable error. | +| `p1-must-env-var` | MUST | Universal | `p1-env-hints` (behavioral)
`p1-env-flags-source` (source) | Every flag settable via environment variable (falsey-value parser for booleans). | +| `p1-must-no-interactive` | MUST | Universal | `p1-non-interactive` (behavioral)
`p1-flag-existence` (behavioral)
`p1-non-interactive-source` (project) | `--no-interactive` flag gates every prompt library call; when set or stdin is not a TTY, use defaults/stdin or exit with an actionable error. | | `p1-must-no-browser` | MUST | If: CLI authenticates against a remote service | `p1-headless-auth` (source) | Headless authentication path (`--no-browser` / OAuth Device Authorization Grant). | | `p1-should-tty-detection` | SHOULD | Universal | `p1-tty-detection-source` (source) | Auto-detect non-interactive context via TTY detection; suppress prompts when stderr is not a terminal. | | `p1-should-defaults-in-help` | SHOULD | Universal | **UNCOVERED** | Document default values for prompted inputs in `--help` output. | @@ -73,7 +74,7 @@ When a requirement has no verifier, the cell reads **UNCOVERED** and the reader | `p6-must-no-color` | MUST | Universal | `p6-no-color-behavioral` (behavioral)
`p6-no-color` (source)
`p6-no-color` (source) | TTY detection plus support for `NO_COLOR` and `TERM=dumb` — color codes suppressed when stdout/stderr is not a terminal. | | `p6-must-completions` | MUST | Universal | `p6-completions` (project) | Shell completions available via a `completions` subcommand (Tier 1 meta-command — needs no config/auth/network). | | `p6-must-timeout-network` | MUST | If: CLI makes network calls | `p6-timeout` (source) | Network CLIs ship a `--timeout` flag with a sensible default (e.g., 30 seconds). | -| `p6-must-no-pager` | MUST | If: CLI invokes a pager for output | `p6-no-pager` (source) | If the CLI uses a pager (`less`, `more`, `$PAGER`), it supports `--no-pager` or respects `PAGER=""`. | +| `p6-must-no-pager` | MUST | If: CLI invokes a pager for output | `p6-no-pager-behavioral` (behavioral)
`p6-no-pager` (source) | If the CLI uses a pager (`less`, `more`, `$PAGER`), it supports `--no-pager` or respects `PAGER=""`. | | `p6-must-global-flags` | MUST | If: CLI uses subcommands | `p6-global-flags` (source) | Agentic flags (`--output`, `--quiet`, `--no-interactive`, `--timeout`) are `global = true` so they propagate to every subcommand. | | `p6-should-stdin-input` | SHOULD | If: CLI has commands that accept input data | **UNCOVERED** | Commands that accept input read from stdin when no file argument is provided. | | `p6-should-consistent-naming` | SHOULD | If: CLI uses subcommands | **UNCOVERED** | Subcommand naming follows a consistent `noun verb` or `verb noun` convention throughout the tool. | diff --git a/src/checks/behavioral/bad_args.rs b/src/checks/behavioral/bad_args.rs index 0b6c983..273189f 100644 --- a/src/checks/behavioral/bad_args.rs +++ b/src/checks/behavioral/bad_args.rs @@ -1,7 +1,7 @@ use crate::check::Check; use crate::project::Project; use crate::runner::RunStatus; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; pub struct BadArgsCheck; @@ -50,6 +50,7 @@ impl Check for BadArgsCheck { group: CheckGroup::P4, layer: CheckLayer::Behavioral, status, + confidence: Confidence::High, }) } } diff --git a/src/checks/behavioral/env_hints.rs b/src/checks/behavioral/env_hints.rs new file mode 100644 index 0000000..d59a92a --- /dev/null +++ b/src/checks/behavioral/env_hints.rs @@ -0,0 +1,150 @@ +//! Check: `--help` advertises environment-variable bindings for flags. +//! +//! Covers: `p1-must-env-var`. Source-verified coverage already exists via +//! `p1-env-flags-source`; this check adds the behavioral layer by inspecting +//! the shipped `--help` surface. Heuristic (Medium confidence): it reads +//! clap-style `[env: FOO]` annotations, which are the canonical but not the +//! only way tools advertise env bindings. +//! +//! Skip when there are no flags at all — a tool with no flags has nothing +//! to bind to env vars. Warn when flags exist but no bindings are visible. + +use crate::check::Check; +use crate::project::Project; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; + +pub struct EnvHintsCheck; + +impl Check for EnvHintsCheck { + fn id(&self) -> &str { + "p1-env-hints" + } + + fn group(&self) -> CheckGroup { + CheckGroup::P1 + } + + fn layer(&self) -> CheckLayer { + CheckLayer::Behavioral + } + + fn covers(&self) -> &'static [&'static str] { + &["p1-must-env-var"] + } + + fn applicable(&self, project: &Project) -> bool { + project.runner.is_some() + } + + fn run(&self, project: &Project) -> anyhow::Result { + let status = match project.help_output() { + None => CheckStatus::Skip("could not probe --help".into()), + Some(help) => check_env_hints(help.flags().len(), help.env_hints().len()), + }; + + Ok(CheckResult { + id: self.id().to_string(), + label: "Flags advertise env-var bindings in --help".into(), + group: self.group(), + layer: self.layer(), + status, + confidence: Confidence::Medium, + }) + } +} + +/// Core unit. Takes parsed-flag count and parsed-env-hint count and returns +/// the `CheckStatus` that summarizes them. +fn check_env_hints(flag_count: usize, env_hint_count: usize) -> CheckStatus { + if flag_count == 0 { + return CheckStatus::Skip("target exposes no flags in --help".into()); + } + if env_hint_count > 0 { + CheckStatus::Pass + } else { + CheckStatus::Warn(format!( + "{flag_count} flag(s) found in --help but no `[env: NAME]` bindings advertised" + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::runner::HelpOutput; + + const HELP_WITH_ENV: &str = r#"Usage: foo [OPTIONS] + +Options: + -q, --quiet Suppress output [env: FOO_QUIET=] + -h, --help Print help +"#; + + const HELP_NO_ENV: &str = r#"Usage: foo [OPTIONS] + +Options: + -q, --quiet Suppress output + -h, --help Print help +"#; + + const HELP_NO_FLAGS: &str = r#"Usage: foo ARG +A tool that takes one positional argument. +"#; + + // Non-English help: parser returns zero env hints and zero flags when + // English conventions don't appear. Per the coverage-matrix exception, + // this is documented English-only behavior. + const HELP_NON_ENGLISH: &str = r#"用法: outil URL + +参数: + URL 目标 +"#; + + #[test] + fn happy_path_env_hint_present() { + let help = HelpOutput::from_raw(HELP_WITH_ENV); + let status = check_env_hints(help.flags().len(), help.env_hints().len()); + assert_eq!(status, CheckStatus::Pass); + } + + #[test] + fn skip_when_no_flags() { + let help = HelpOutput::from_raw(HELP_NO_FLAGS); + let status = check_env_hints(help.flags().len(), help.env_hints().len()); + assert!(matches!(status, CheckStatus::Skip(_))); + } + + #[test] + fn warn_when_flags_but_no_env_hints() { + let help = HelpOutput::from_raw(HELP_NO_ENV); + let status = check_env_hints(help.flags().len(), help.env_hints().len()); + match status { + CheckStatus::Warn(msg) => { + assert!(msg.contains("env")); + assert!(msg.contains("flag")); + } + other => panic!("expected Warn, got {other:?}"), + } + } + + #[test] + fn non_english_help_skipped_or_warned() { + // Localized help with no ASCII options block — parsers return empty + // flags + empty env_hints. Skip (no flags to bind). + let help = HelpOutput::from_raw(HELP_NON_ENGLISH); + let status = check_env_hints(help.flags().len(), help.env_hints().len()); + assert!(matches!(status, CheckStatus::Skip(_))); + } + + #[test] + fn unit_core_returns_pass_with_any_hint() { + assert_eq!(check_env_hints(3, 1), CheckStatus::Pass); + assert_eq!(check_env_hints(3, 10), CheckStatus::Pass); + } + + #[test] + fn unit_core_warns_when_zero_hints() { + let status = check_env_hints(5, 0); + assert!(matches!(status, CheckStatus::Warn(_))); + } +} diff --git a/src/checks/behavioral/flag_existence.rs b/src/checks/behavioral/flag_existence.rs new file mode 100644 index 0000000..caf2252 --- /dev/null +++ b/src/checks/behavioral/flag_existence.rs @@ -0,0 +1,247 @@ +//! Check: `--help` advertises at least one non-interactive gate flag. +//! +//! Covers: `p1-must-no-interactive`. This is the second behavioral proof +//! of the same MUST — the existing `p1-non-interactive` check probes +//! *runtime* behavior (bare invocation, stdin-primary). This check probes +//! the *flag surface area* — does the CLI advertise any of the canonical +//! non-interactive flags (`--no-interactive`, `-p`, `--batch`, ...) in +//! its `--help` output at all. +//! +//! Skip rather than Warn when the target already satisfies P1 via an +//! alternative gate (help-on-bare-invocation or stdin-clean-exit) — those +//! tools don't need an advertised flag to be agent-safe. + +use crate::check::Check; +use crate::project::Project; +use crate::runner::RunStatus; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; + +/// Canonical non-interactive gate flags. A tool that advertises any one +/// of these in `--help` is explicitly agent-addressable. Kept narrow on +/// purpose — broader matching produces false positives on tools where +/// `-y` means "yes file format" and similar collisions. +const GATE_FLAGS: &[&str] = &[ + "--no-interactive", + "--non-interactive", + "-p", + "--print", + "--no-input", + "--batch", + "--headless", + "-y", + "--yes", + "--assume-yes", +]; + +const HELP_ON_BARE_MARKERS: &[&str] = &["Usage:", "USAGE:", "usage:"]; + +pub struct FlagExistenceCheck; + +impl Check for FlagExistenceCheck { + fn id(&self) -> &str { + "p1-flag-existence" + } + + fn group(&self) -> CheckGroup { + CheckGroup::P1 + } + + fn layer(&self) -> CheckLayer { + CheckLayer::Behavioral + } + + fn covers(&self) -> &'static [&'static str] { + &["p1-must-no-interactive"] + } + + fn applicable(&self, project: &Project) -> bool { + project.runner.is_some() + } + + fn run(&self, project: &Project) -> anyhow::Result { + let runner = project.runner_ref(); + + // Skip when the target already satisfies P1 via an alternative gate. + // These probes hit BinaryRunner's cache when `p1-non-interactive` + // already ran, so the cost is zero. + let bare = runner.run(&[], &[]); + let bare_output = format!("{}{}", bare.stdout, bare.stderr); + let help_on_bare = HELP_ON_BARE_MARKERS.iter().any(|m| bare_output.contains(m)); + let stdin_clean_exit = matches!(bare.status, RunStatus::Ok); + + if help_on_bare || stdin_clean_exit { + return Ok(CheckResult { + id: self.id().to_string(), + label: "Non-interactive gate flag advertised in --help".into(), + group: self.group(), + layer: self.layer(), + status: CheckStatus::Skip( + "target satisfies P1 via alternative gate (help-on-bare or stdin-primary)" + .into(), + ), + confidence: Confidence::High, + }); + } + + let status = match project.help_output() { + None => CheckStatus::Skip("could not probe --help".into()), + Some(help) => { + let raw = help.raw(); + if raw.trim().is_empty() { + CheckStatus::Skip( + "--help produced no output (likely non-English or unsupported)".into(), + ) + } else if GATE_FLAGS.iter().any(|needle| contains_flag(raw, needle)) { + CheckStatus::Pass + } else { + CheckStatus::Warn(format!( + "no non-interactive flag found in --help; expected one of: {}", + GATE_FLAGS.join(", ") + )) + } + } + }; + + Ok(CheckResult { + id: self.id().to_string(), + label: "Non-interactive gate flag advertised in --help".into(), + group: self.group(), + layer: self.layer(), + status, + confidence: Confidence::High, + }) + } +} + +/// A flag needle like `"--no-interactive"` matches when it appears in the +/// help text bounded by a non-flag character on either side — so `--no-input` +/// does not satisfy a search for `-p`, and `--print-json` does not satisfy +/// a search for `--print`. +fn contains_flag(haystack: &str, needle: &str) -> bool { + let mut rest = haystack; + while let Some(pos) = rest.find(needle) { + let before_ok = pos == 0 || !is_flag_name_char(rest.as_bytes()[pos - 1] as char); + let after_idx = pos + needle.len(); + let after_ok = + after_idx >= rest.len() || !is_flag_name_char(rest.as_bytes()[after_idx] as char); + if before_ok && after_ok { + return true; + } + rest = &rest[after_idx..]; + } + false +} + +fn is_flag_name_char(c: char) -> bool { + c.is_ascii_alphanumeric() || c == '-' || c == '_' +} + +/// Core unit for tests. Takes pre-captured help text and the bare-invocation +/// status, returns a `CheckStatus` — mirrors the check convention in +/// `CLAUDE.md` §"Source Check Convention" (adapted for behavioral checks). +#[cfg(test)] +fn check_flag_existence(help_raw: &str, bare_stdout: &str, bare_ok: bool) -> CheckStatus { + let help_on_bare = HELP_ON_BARE_MARKERS.iter().any(|m| bare_stdout.contains(m)); + if help_on_bare || bare_ok { + return CheckStatus::Skip( + "target satisfies P1 via alternative gate (help-on-bare or stdin-primary)".into(), + ); + } + if help_raw.trim().is_empty() { + return CheckStatus::Skip( + "--help produced no output (likely non-English or unsupported)".into(), + ); + } + if GATE_FLAGS + .iter() + .any(|needle| contains_flag(help_raw, needle)) + { + CheckStatus::Pass + } else { + CheckStatus::Warn(format!( + "no non-interactive flag found in --help; expected one of: {}", + GATE_FLAGS.join(", ") + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn happy_path_batch_flag_in_help() { + // --help advertises `--batch` → Pass. Bare invocation did NOT print + // Usage and did NOT exit cleanly, so the alternative gates do not fire. + let help = " --batch Run in batch mode.\n"; + assert_eq!(check_flag_existence(help, "", false), CheckStatus::Pass); + } + + #[test] + fn happy_path_short_print_flag() { + let help = " -p, --print Print output.\n"; + assert_eq!(check_flag_existence(help, "", false), CheckStatus::Pass); + } + + #[test] + fn skip_when_help_on_bare_invocation() { + // Bare invocation printed Usage → tool is already agent-safe. + let help = " --foo Do a thing.\n"; + let result = check_flag_existence(help, "Usage: foo [OPTIONS]\n", false); + assert!(matches!(result, CheckStatus::Skip(_))); + } + + #[test] + fn skip_when_stdin_clean_exit() { + // Bare invocation exited 0 (stdin-primary behavior) → skip. + let help = " --foo Do a thing.\n"; + let result = check_flag_existence(help, "", true); + assert!(matches!(result, CheckStatus::Skip(_))); + } + + #[test] + fn warn_when_no_gate_flag_and_no_alt_gate() { + let help = " --color When to color.\n --version Print version.\n"; + match check_flag_existence(help, "", false) { + CheckStatus::Warn(msg) => assert!(msg.contains("--no-interactive")), + other => panic!("expected Warn, got {other:?}"), + } + } + + #[test] + fn non_english_help_is_skipped() { + // Localized help without any English flag text → empty input after + // we strip non-ASCII — the check honors the English-only regex + // exception from docs/coverage-matrix.md. The parsers would still + // return zero matches, so we Warn. For "completely unparseable" + // localized help with no ASCII flags at all, we Skip via the empty + // branch. Exercise both. + // Completely non-English: no flags detected → warn about missing flag. + let help = "用法: outil\n选项:\n -H, --header 自定义请求头\n"; + let result = check_flag_existence(help, "", false); + assert!(matches!(result, CheckStatus::Warn(_))); + + // Empty help output → Skip. + let empty = ""; + let result = check_flag_existence(empty, "", false); + assert!(matches!(result, CheckStatus::Skip(_))); + } + + #[test] + fn word_boundary_rejects_partial_matches() { + // `--print-json` must NOT satisfy a search for `--print` — but + // `--print` alone in a neighbor line must. + let help = " --print-json Print as JSON.\n"; + let result = check_flag_existence(help, "", false); + assert!(matches!(result, CheckStatus::Warn(_))); + } + + #[test] + fn contains_flag_word_boundary() { + assert!(contains_flag("use --batch mode", "--batch")); + assert!(contains_flag(" --batch\n", "--batch")); + assert!(!contains_flag("--batching", "--batch")); + assert!(contains_flag("-p, --print", "-p")); + assert!(!contains_flag("-pr", "-p")); + } +} diff --git a/src/checks/behavioral/help.rs b/src/checks/behavioral/help.rs index 41da62e..324c310 100644 --- a/src/checks/behavioral/help.rs +++ b/src/checks/behavioral/help.rs @@ -1,7 +1,7 @@ use crate::check::Check; use crate::project::Project; use crate::runner::RunStatus; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; pub struct HelpCheck; @@ -59,6 +59,7 @@ impl Check for HelpCheck { group: CheckGroup::P3, layer: CheckLayer::Behavioral, status, + confidence: Confidence::High, }) } } diff --git a/src/checks/behavioral/json_output.rs b/src/checks/behavioral/json_output.rs index 26246c3..fb02cba 100644 --- a/src/checks/behavioral/json_output.rs +++ b/src/checks/behavioral/json_output.rs @@ -1,7 +1,7 @@ use crate::check::Check; use crate::project::Project; use crate::runner::{BinaryRunner, RunStatus}; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; pub struct JsonOutputCheck; @@ -55,6 +55,7 @@ impl Check for JsonOutputCheck { group: CheckGroup::P2, layer: CheckLayer::Behavioral, status, + confidence: Confidence::High, }) } } diff --git a/src/checks/behavioral/mod.rs b/src/checks/behavioral/mod.rs index ec77a66..1461994 100644 --- a/src/checks/behavioral/mod.rs +++ b/src/checks/behavioral/mod.rs @@ -1,7 +1,10 @@ mod bad_args; +mod env_hints; +mod flag_existence; mod help; mod json_output; mod no_color; +mod no_pager_behavioral; mod non_interactive; mod quiet; mod sigpipe; @@ -18,6 +21,9 @@ pub fn all_behavioral_checks() -> Vec> { Box::new(quiet::QuietCheck), Box::new(sigpipe::SigpipeCheck), Box::new(non_interactive::NonInteractiveCheck), + Box::new(flag_existence::FlagExistenceCheck), + Box::new(env_hints::EnvHintsCheck), + Box::new(no_pager_behavioral::NoPagerBehavioralCheck), Box::new(no_color::NoColorBehavioralCheck), ] } @@ -44,6 +50,7 @@ pub(crate) mod tests { ), include_tests: false, parsed_files: OnceLock::new(), + help_output: OnceLock::new(), } } @@ -103,6 +110,7 @@ pub(crate) mod tests { ), include_tests: false, parsed_files: OnceLock::new(), + help_output: OnceLock::new(), } } } diff --git a/src/checks/behavioral/no_color.rs b/src/checks/behavioral/no_color.rs index 0724d0b..791107d 100644 --- a/src/checks/behavioral/no_color.rs +++ b/src/checks/behavioral/no_color.rs @@ -1,7 +1,7 @@ use crate::check::Check; use crate::project::Project; use crate::runner::RunStatus; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; pub struct NoColorBehavioralCheck; @@ -51,6 +51,7 @@ impl Check for NoColorBehavioralCheck { group: CheckGroup::P6, layer: CheckLayer::Behavioral, status, + confidence: Confidence::High, }) } } diff --git a/src/checks/behavioral/no_pager_behavioral.rs b/src/checks/behavioral/no_pager_behavioral.rs new file mode 100644 index 0000000..e90b987 --- /dev/null +++ b/src/checks/behavioral/no_pager_behavioral.rs @@ -0,0 +1,152 @@ +//! Check: behavioral confirmation that a pager-using tool ships `--no-pager`. +//! +//! Covers: `p6-must-no-pager`. Source-verified coverage already exists via +//! `p6-no-pager`; this check adds the behavioral layer by inspecting the +//! shipped `--help` surface. Heuristic (Medium confidence): pager inference +//! from the text is soft. +//! +//! Pass when `--no-pager` is in the advertised flag list. Skip when the +//! help text shows no pager signal at all (nothing mentions `less`, `more`, +//! `$PAGER`, `--pager`, or `pager`). Warn when the text mentions pager +//! plumbing but the escape hatch is absent. + +use crate::check::Check; +use crate::project::Project; +use crate::runner::HelpOutput; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; + +const PAGER_SIGNALS: &[&str] = &["less", "more", "$PAGER", "--pager", "pager", "PAGER"]; + +pub struct NoPagerBehavioralCheck; + +impl Check for NoPagerBehavioralCheck { + fn id(&self) -> &str { + "p6-no-pager-behavioral" + } + + fn group(&self) -> CheckGroup { + CheckGroup::P6 + } + + fn layer(&self) -> CheckLayer { + CheckLayer::Behavioral + } + + fn covers(&self) -> &'static [&'static str] { + &["p6-must-no-pager"] + } + + fn applicable(&self, project: &Project) -> bool { + project.runner.is_some() + } + + fn run(&self, project: &Project) -> anyhow::Result { + let status = match project.help_output() { + None => CheckStatus::Skip("could not probe --help".into()), + Some(help) => check_no_pager(help), + }; + + Ok(CheckResult { + id: self.id().to_string(), + label: "Pager-using CLI ships --no-pager escape hatch".into(), + group: self.group(), + layer: self.layer(), + status, + confidence: Confidence::Medium, + }) + } +} + +/// Core unit for tests. Takes a prepared `HelpOutput` and returns the +/// `CheckStatus` that summarizes it. +fn check_no_pager(help: &HelpOutput) -> CheckStatus { + let has_no_pager_flag = help.flags().iter().any(|f| f.matches("--no-pager")); + if has_no_pager_flag { + return CheckStatus::Pass; + } + let raw = help.raw(); + let mentions_pager = PAGER_SIGNALS.iter().any(|sig| raw.contains(sig)); + if !mentions_pager { + CheckStatus::Skip("no pager signal (less/more/$PAGER/--pager) in --help".into()) + } else { + CheckStatus::Warn( + "pager referenced in --help but no --no-pager escape hatch advertised".into(), + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const HELP_WITH_NO_PAGER: &str = r#"Usage: tool [OPTIONS] + +Options: + --no-pager Disable paged output. + --pager Use custom pager. + -h, --help Show help. +"#; + + const HELP_PAGER_WITHOUT_ESCAPE: &str = r#"Usage: tool [OPTIONS] + +Long output is piped through less by default. Set $PAGER to override. + +Options: + -h, --help Show help. +"#; + + const HELP_NO_PAGER_MENTION: &str = r#"Usage: tool [OPTIONS] + +Options: + -q, --quiet Suppress output. + -h, --help Show help. +"#; + + const HELP_NON_ENGLISH: &str = r#"用法: outil [选项] + +选项: + -h, --help 显示帮助 +"#; + + #[test] + fn happy_path_no_pager_flag_present() { + let help = HelpOutput::from_raw(HELP_WITH_NO_PAGER); + assert_eq!(check_no_pager(&help), CheckStatus::Pass); + } + + #[test] + fn skip_when_no_pager_mention_at_all() { + let help = HelpOutput::from_raw(HELP_NO_PAGER_MENTION); + let status = check_no_pager(&help); + assert!(matches!(status, CheckStatus::Skip(_))); + } + + #[test] + fn warn_when_pager_mentioned_without_escape() { + let help = HelpOutput::from_raw(HELP_PAGER_WITHOUT_ESCAPE); + let status = check_no_pager(&help); + match status { + CheckStatus::Warn(msg) => { + assert!(msg.contains("--no-pager")); + } + other => panic!("expected Warn, got {other:?}"), + } + } + + #[test] + fn non_english_help_skipped() { + // Localized help with no pager-adjacent ASCII tokens → Skip via + // "no signal" branch. The English-only exception is documented. + let help = HelpOutput::from_raw(HELP_NON_ENGLISH); + let status = check_no_pager(&help); + assert!(matches!(status, CheckStatus::Skip(_))); + } + + #[test] + fn detects_no_pager_with_mixed_casing() { + // Ensure we match `--no-pager` as a long flag regardless of how the + // `Flag::matches` helper receives the query. + let help = HelpOutput::from_raw(" --no-pager Disable paging.\n"); + assert_eq!(check_no_pager(&help), CheckStatus::Pass); + } +} diff --git a/src/checks/behavioral/non_interactive.rs b/src/checks/behavioral/non_interactive.rs index 59fc6c4..125fc96 100644 --- a/src/checks/behavioral/non_interactive.rs +++ b/src/checks/behavioral/non_interactive.rs @@ -1,7 +1,7 @@ use crate::check::Check; use crate::project::Project; use crate::runner::RunStatus; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; /// Agentic flag markers in `--help` output that signal the tool exposes a /// headless path. Matching any one satisfies P1's "no blocking-interactive @@ -102,6 +102,7 @@ impl Check for NonInteractiveCheck { group: CheckGroup::P1, layer: CheckLayer::Behavioral, status, + confidence: Confidence::High, }) } } diff --git a/src/checks/behavioral/quiet.rs b/src/checks/behavioral/quiet.rs index 2058f8f..f4d89f8 100644 --- a/src/checks/behavioral/quiet.rs +++ b/src/checks/behavioral/quiet.rs @@ -1,7 +1,7 @@ use crate::check::Check; use crate::project::Project; use crate::runner::RunStatus; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; pub struct QuietCheck; @@ -48,6 +48,7 @@ impl Check for QuietCheck { group: CheckGroup::P7, layer: CheckLayer::Behavioral, status, + confidence: Confidence::High, }) } } diff --git a/src/checks/behavioral/sigpipe.rs b/src/checks/behavioral/sigpipe.rs index cf42351..4255fee 100644 --- a/src/checks/behavioral/sigpipe.rs +++ b/src/checks/behavioral/sigpipe.rs @@ -1,7 +1,7 @@ use crate::check::Check; use crate::project::Project; use crate::runner::RunStatus; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; pub struct SigpipeCheck; @@ -44,6 +44,7 @@ impl Check for SigpipeCheck { group: CheckGroup::P6, layer: CheckLayer::Behavioral, status, + confidence: Confidence::High, }) } } diff --git a/src/checks/behavioral/version.rs b/src/checks/behavioral/version.rs index 2a7899e..d504016 100644 --- a/src/checks/behavioral/version.rs +++ b/src/checks/behavioral/version.rs @@ -1,7 +1,7 @@ use crate::check::Check; use crate::project::Project; use crate::runner::RunStatus; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; pub struct VersionCheck; @@ -46,6 +46,7 @@ impl Check for VersionCheck { group: CheckGroup::P3, layer: CheckLayer::Behavioral, status, + confidence: Confidence::High, }) } } diff --git a/src/checks/project/agents_md.rs b/src/checks/project/agents_md.rs index 06b2c31..dbd9f0c 100644 --- a/src/checks/project/agents_md.rs +++ b/src/checks/project/agents_md.rs @@ -5,7 +5,7 @@ use crate::check::Check; use crate::project::Project; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; pub struct AgentsMdCheck; @@ -41,6 +41,7 @@ impl Check for AgentsMdCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/project/completions.rs b/src/checks/project/completions.rs index 6873b34..81d075f 100644 --- a/src/checks/project/completions.rs +++ b/src/checks/project/completions.rs @@ -7,7 +7,7 @@ use std::fs; use crate::check::Check; use crate::project::{Language, Project}; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; pub struct CompletionsCheck; @@ -60,6 +60,7 @@ impl Check for CompletionsCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/project/dependencies.rs b/src/checks/project/dependencies.rs index 695038b..99b6407 100644 --- a/src/checks/project/dependencies.rs +++ b/src/checks/project/dependencies.rs @@ -8,7 +8,7 @@ use std::fs; use crate::check::Check; use crate::project::{Language, Project}; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; /// Recommended dependency groups: (description, alternatives). /// A group passes if any alternative is present. @@ -71,6 +71,7 @@ impl Check for DependenciesCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/project/dry_run.rs b/src/checks/project/dry_run.rs index a1dbe5f..c567d95 100644 --- a/src/checks/project/dry_run.rs +++ b/src/checks/project/dry_run.rs @@ -13,7 +13,7 @@ use std::fs; use crate::check::Check; use crate::project::Project; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; /// Keywords in clap arg definitions that indicate write/mutate operations. const WRITE_KEYWORDS: &[&str] = &[ @@ -103,6 +103,7 @@ impl Check for DryRunCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } @@ -159,6 +160,7 @@ mod tests { runner: None, include_tests: false, parsed_files: OnceLock::from(parsed), + help_output: OnceLock::new(), } } diff --git a/src/checks/project/error_module.rs b/src/checks/project/error_module.rs index aaac36e..98a004c 100644 --- a/src/checks/project/error_module.rs +++ b/src/checks/project/error_module.rs @@ -7,7 +7,7 @@ use std::fs; use crate::check::Check; use crate::project::Project; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; pub struct ErrorModuleCheck; @@ -45,6 +45,7 @@ impl Check for ErrorModuleCheck { group: self.group(), layer: self.layer(), status: CheckStatus::Pass, + confidence: Confidence::High, }); } } @@ -63,6 +64,7 @@ impl Check for ErrorModuleCheck { group: self.group(), layer: self.layer(), status: CheckStatus::Pass, + confidence: Confidence::High, }); } } @@ -79,6 +81,7 @@ impl Check for ErrorModuleCheck { status: CheckStatus::Warn( "No dedicated error module found (expected src/error.rs or src/errors.rs)".into(), ), + confidence: Confidence::High, }) } } diff --git a/src/checks/project/non_interactive.rs b/src/checks/project/non_interactive.rs index bef1849..4ef0208 100644 --- a/src/checks/project/non_interactive.rs +++ b/src/checks/project/non_interactive.rs @@ -7,7 +7,7 @@ use std::fs; use crate::check::Check; use crate::project::{Language, Project}; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; /// Interactive prompt libraries that conflict with agent-native operation. const PROMPT_LIBS: &[&str] = &["dialoguer", "inquire", "rustyline", "crossterm"]; @@ -66,6 +66,7 @@ impl Check for NonInteractiveSourceCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/source/python/bare_except.rs b/src/checks/source/python/bare_except.rs index 47cfb2d..6687f6a 100644 --- a/src/checks/source/python/bare_except.rs +++ b/src/checks/source/python/bare_except.rs @@ -9,7 +9,7 @@ use ast_grep_language::Python; use crate::check::Check; use crate::project::{Language, Project}; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, SourceLocation}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence, SourceLocation}; /// Check trait implementation for bare-except detection. pub struct BareExceptCheck; @@ -54,6 +54,7 @@ impl Check for BareExceptCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/source/python/no_color.rs b/src/checks/source/python/no_color.rs index 319ead8..478b17e 100644 --- a/src/checks/source/python/no_color.rs +++ b/src/checks/source/python/no_color.rs @@ -14,7 +14,7 @@ use ast_grep_language::Python; use crate::check::Check; use crate::project::{Language, Project}; use crate::source::has_string_literal_in; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; /// Check trait implementation for NO_COLOR detection in Python. pub struct NoColorPythonCheck; @@ -68,6 +68,7 @@ impl Check for NoColorPythonCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/source/python/sys_exit.rs b/src/checks/source/python/sys_exit.rs index 71af9c4..b60e42d 100644 --- a/src/checks/source/python/sys_exit.rs +++ b/src/checks/source/python/sys_exit.rs @@ -10,7 +10,7 @@ use ast_grep_language::Python; use crate::check::Check; use crate::project::{Language, Project}; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, SourceLocation}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence, SourceLocation}; /// Check trait implementation for sys.exit() outside __main__ guard. pub struct SysExitCheck; @@ -60,6 +60,7 @@ impl Check for SysExitCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/source/rust/env_flags.rs b/src/checks/source/rust/env_flags.rs index 718cc29..1a6efc5 100644 --- a/src/checks/source/rust/env_flags.rs +++ b/src/checks/source/rust/env_flags.rs @@ -12,7 +12,7 @@ use ast_grep_language::Rust; use crate::check::Check; use crate::project::{Language, Project}; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, SourceLocation}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence, SourceLocation}; /// Agentic flags that should have `env = "..."` backing. const AGENTIC_FLAGS: &[&str] = &[ @@ -79,6 +79,7 @@ impl Check for EnvFlagsCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/source/rust/error_types.rs b/src/checks/source/rust/error_types.rs index 8d45ae9..9c570ea 100644 --- a/src/checks/source/rust/error_types.rs +++ b/src/checks/source/rust/error_types.rs @@ -11,7 +11,7 @@ use ast_grep_language::Rust; use crate::check::Check; use crate::project::{Language, Project}; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; /// Check trait implementation for structured error type detection. pub struct ErrorTypesCheck; @@ -64,6 +64,7 @@ impl Check for ErrorTypesCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/source/rust/exit_codes.rs b/src/checks/source/rust/exit_codes.rs index 62d5f9c..da023fc 100644 --- a/src/checks/source/rust/exit_codes.rs +++ b/src/checks/source/rust/exit_codes.rs @@ -12,7 +12,7 @@ use ast_grep_language::Rust; use crate::check::Check; use crate::project::{Language, Project}; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, SourceLocation}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence, SourceLocation}; /// Check trait implementation for raw exit code detection. pub struct ExitCodesCheck; @@ -61,6 +61,7 @@ impl Check for ExitCodesCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/source/rust/global_flags.rs b/src/checks/source/rust/global_flags.rs index fb0ce49..9671f11 100644 --- a/src/checks/source/rust/global_flags.rs +++ b/src/checks/source/rust/global_flags.rs @@ -15,7 +15,7 @@ use ast_grep_language::Rust; use crate::check::Check; use crate::project::{Language, Project}; use crate::source::has_pattern; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, SourceLocation}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence, SourceLocation}; /// Agentic flags that should be global when subcommands exist. const AGENTIC_FLAGS: &[&str] = &["output", "quiet", "verbose", "no_color", "no-color"]; @@ -81,6 +81,7 @@ impl Check for GlobalFlagsCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/source/rust/headless_auth.rs b/src/checks/source/rust/headless_auth.rs index f489e70..38ba7b9 100644 --- a/src/checks/source/rust/headless_auth.rs +++ b/src/checks/source/rust/headless_auth.rs @@ -10,7 +10,7 @@ use crate::check::Check; use crate::project::{Language, Project}; use crate::source::has_pattern; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; /// Auth-related substrings to search for in Rust identifiers (not string /// literals or comments). We search function definitions via ast-grep to @@ -89,6 +89,7 @@ impl Check for HeadlessAuthCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/source/rust/naked_println.rs b/src/checks/source/rust/naked_println.rs index 362cb46..3713c07 100644 --- a/src/checks/source/rust/naked_println.rs +++ b/src/checks/source/rust/naked_println.rs @@ -8,7 +8,7 @@ use crate::check::Check; use crate::project::{Language, Project}; use crate::source::find_pattern_matches; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; const PRINTLN_PATTERN: &str = "println!($$$ARGS)"; const PRINT_PATTERN: &str = "print!($$$ARGS)"; @@ -64,6 +64,7 @@ impl Check for NakedPrintlnCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/source/rust/no_color.rs b/src/checks/source/rust/no_color.rs index 2cef2d6..3bdd04f 100644 --- a/src/checks/source/rust/no_color.rs +++ b/src/checks/source/rust/no_color.rs @@ -14,7 +14,7 @@ use ast_grep_language::Rust; use crate::check::Check; use crate::project::{Language, Project}; use crate::source::{has_pattern, has_string_literal_in}; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; /// Check trait implementation for NO_COLOR detection. pub struct NoColorSourceCheck; @@ -67,6 +67,7 @@ impl Check for NoColorSourceCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/source/rust/no_pager.rs b/src/checks/source/rust/no_pager.rs index a965184..2de0482 100644 --- a/src/checks/source/rust/no_pager.rs +++ b/src/checks/source/rust/no_pager.rs @@ -10,7 +10,7 @@ use crate::check::Check; use crate::project::{Language, Project}; use crate::source::has_pattern; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; /// Pager-related patterns to detect in source text. const PAGER_INDICATORS: &[&str] = &["pager::Pager", "Pager::new", "Pager::with_pager"]; @@ -74,6 +74,7 @@ impl Check for NoPagerCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/source/rust/output_clamping.rs b/src/checks/source/rust/output_clamping.rs index b9ef7d7..d2de96f 100644 --- a/src/checks/source/rust/output_clamping.rs +++ b/src/checks/source/rust/output_clamping.rs @@ -9,7 +9,7 @@ use crate::check::Check; use crate::project::{Language, Project}; use crate::source::{find_pattern_matches, has_pattern}; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; /// Patterns that suggest list/collection output. const LIST_PATTERNS: &[&str] = &[ @@ -86,6 +86,7 @@ impl Check for OutputClampingCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/source/rust/output_module.rs b/src/checks/source/rust/output_module.rs index c3b231c..0e55b8b 100644 --- a/src/checks/source/rust/output_module.rs +++ b/src/checks/source/rust/output_module.rs @@ -11,7 +11,7 @@ use std::path::Path; use crate::check::Check; use crate::project::{Language, Project}; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; // Rust source check — lives in source/rust because the content patterns // (fn format_, impl Display, std::fmt::Write) are Rust-specific. @@ -54,6 +54,7 @@ impl Check for OutputModuleCheck { group: self.group(), layer: self.layer(), status: CheckStatus::Pass, + confidence: Confidence::High, }); } } @@ -68,6 +69,7 @@ impl Check for OutputModuleCheck { with format/render/display functions rather than scattering print calls." .into(), ), + confidence: Confidence::High, }) } } diff --git a/src/checks/source/rust/process_exit.rs b/src/checks/source/rust/process_exit.rs index b4ece48..8287abc 100644 --- a/src/checks/source/rust/process_exit.rs +++ b/src/checks/source/rust/process_exit.rs @@ -6,7 +6,7 @@ use crate::check::Check; use crate::project::{Language, Project}; use crate::source::find_pattern_matches; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; const PATTERNS: &[&str] = &["process::exit($CODE)", "std::process::exit($CODE)"]; @@ -59,6 +59,7 @@ impl Check for ProcessExitCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/source/rust/structured_output.rs b/src/checks/source/rust/structured_output.rs index 396aea9..cb5a056 100644 --- a/src/checks/source/rust/structured_output.rs +++ b/src/checks/source/rust/structured_output.rs @@ -9,7 +9,7 @@ use crate::check::Check; use crate::project::{Language, Project}; use crate::source::has_pattern; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; /// Check trait implementation for structured output detection. pub struct StructuredOutputCheck; @@ -74,6 +74,7 @@ impl Check for StructuredOutputCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/source/rust/timeout_flag.rs b/src/checks/source/rust/timeout_flag.rs index 4e02849..207cb05 100644 --- a/src/checks/source/rust/timeout_flag.rs +++ b/src/checks/source/rust/timeout_flag.rs @@ -9,7 +9,7 @@ use crate::check::Check; use crate::project::{Language, Project}; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; /// Network library indicators to search for in source text. const NETWORK_INDICATORS: &[&str] = &["reqwest", "hyper", "curl", "ureq"]; @@ -77,6 +77,7 @@ impl Check for TimeoutFlagCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/source/rust/try_parse.rs b/src/checks/source/rust/try_parse.rs index 5e2af44..65750ea 100644 --- a/src/checks/source/rust/try_parse.rs +++ b/src/checks/source/rust/try_parse.rs @@ -9,7 +9,7 @@ use crate::check::Check; use crate::project::{Language, Project}; use crate::source::find_pattern_matches; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; const PATTERNS: &[&str] = &["$RECV.parse().unwrap()"]; @@ -60,6 +60,7 @@ impl Check for TryParseCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/source/rust/tty_detection.rs b/src/checks/source/rust/tty_detection.rs index 98a4639..2715741 100644 --- a/src/checks/source/rust/tty_detection.rs +++ b/src/checks/source/rust/tty_detection.rs @@ -15,7 +15,7 @@ use crate::check::Check; use crate::project::{Language, Project}; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; /// Color/formatting indicators to search for in source text. const COLOR_INDICATORS: &[&str] = &[ @@ -99,6 +99,7 @@ impl Check for TtyDetectionCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/checks/source/rust/unwrap.rs b/src/checks/source/rust/unwrap.rs index 3231c26..b34a615 100644 --- a/src/checks/source/rust/unwrap.rs +++ b/src/checks/source/rust/unwrap.rs @@ -6,7 +6,7 @@ use crate::check::Check; use crate::project::{Language, Project}; use crate::source::find_pattern_matches; -use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; +use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; const PATTERN: &str = "$RECV.unwrap()"; @@ -53,6 +53,7 @@ impl Check for UnwrapCheck { group: self.group(), layer: self.layer(), status, + confidence: Confidence::High, }) } } diff --git a/src/main.rs b/src/main.rs index bf02323..11c98e8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,7 +23,7 @@ use error::AppError; use principles::matrix; use project::Project; use scorecard::{exit_code, format_json, format_text}; -use types::{CheckGroup, CheckResult, CheckStatus}; +use types::{CheckGroup, CheckResult, CheckStatus, Confidence}; fn main() { // Fix SIGPIPE handling so piping to head/grep works correctly. @@ -138,6 +138,7 @@ fn run() -> Result { group: check.group(), layer: check.layer(), status: CheckStatus::Error(e.to_string()), + confidence: Confidence::High, }, }; results.push(result); diff --git a/src/principles/matrix.rs b/src/principles/matrix.rs index 2924404..0c5cbf6 100644 --- a/src/principles/matrix.rs +++ b/src/principles/matrix.rs @@ -50,6 +50,11 @@ pub struct MatrixSummary { pub total: usize, pub covered: usize, pub uncovered: usize, + /// Covered requirements that have at least two verifiers, spanning + /// behavioral + source (or project) layers. Dual-layer coverage is the + /// headline signal that a requirement is pinned down from more than + /// one angle — useful for spotting surface-only verifiers. + pub dual_layer: usize, pub must: LevelSummary, pub should: LevelSummary, pub may: LevelSummary, @@ -114,6 +119,7 @@ fn summarize(rows: &[MatrixRow]) -> MatrixSummary { covered: 0, }; let mut covered = 0; + let mut dual_layer = 0; for row in rows { let bucket = match row.level { @@ -125,6 +131,9 @@ fn summarize(rows: &[MatrixRow]) -> MatrixSummary { if !row.verifiers.is_empty() { bucket.covered += 1; covered += 1; + if row.verifiers.len() >= 2 { + dual_layer += 1; + } } } @@ -132,6 +141,7 @@ fn summarize(rows: &[MatrixRow]) -> MatrixSummary { total: rows.len(), covered, uncovered: rows.len() - covered, + dual_layer, must, should, may, @@ -168,6 +178,11 @@ pub fn render_markdown(matrix: &Matrix) -> String { "- **Total**: {} requirements ({} covered / {} uncovered)", s.total, s.covered, s.uncovered ); + let _ = writeln!( + out, + "- **Dual-layer**: {} of {} covered requirements have verifiers in two layers (behavioral + source or project)", + s.dual_layer, s.covered + ); let _ = writeln!( out, "- **MUST**: {} of {} covered", @@ -284,7 +299,7 @@ mod tests { use super::*; use crate::check::Check; use crate::project::Project; - use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; + use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; struct FakeCheck { id: &'static str, @@ -311,6 +326,7 @@ mod tests { group: CheckGroup::P1, layer: CheckLayer::Behavioral, status: CheckStatus::Pass, + confidence: Confidence::High, }) } fn covers(&self) -> &'static [&'static str] { diff --git a/src/project.rs b/src/project.rs index 4e30fc6..397fa0f 100644 --- a/src/project.rs +++ b/src/project.rs @@ -7,7 +7,7 @@ use std::time::Duration; use anyhow::{Context, Result, bail}; use serde::Serialize; -use crate::runner::BinaryRunner; +use crate::runner::{BinaryRunner, HelpOutput}; /// Maximum directory recursion depth for source file walk. const MAX_DEPTH: usize = 20; @@ -36,6 +36,7 @@ pub struct Project { pub runner: Option, pub include_tests: bool, pub(crate) parsed_files: OnceLock>, + pub(crate) help_output: OnceLock>, } impl std::fmt::Debug for Project { @@ -51,6 +52,7 @@ impl std::fmt::Debug for Project { "parsed_files_count", &self.parsed_files.get().map_or(0, |m| m.len()), ) + .field("help_probed", &self.help_output.get().is_some()) .finish() } } @@ -77,6 +79,7 @@ impl Project { runner, include_tests: false, parsed_files: OnceLock::new(), + help_output: OnceLock::new(), }); } @@ -98,6 +101,7 @@ impl Project { runner, include_tests: false, parsed_files: OnceLock::new(), + help_output: OnceLock::new(), }) } @@ -111,6 +115,20 @@ impl Project { .expect("runner must exist when applicable() returns true") } + /// Lazily probe ` --help` exactly once, returning a shared + /// reference that behavioral checks consume. Returns `None` when the + /// project has no runner or the help probe fails outright (e.g., binary + /// missing). `HelpOutput` itself handles partial captures from timeouts + /// and crashes — those still yield `Some(_)`. + pub fn help_output(&self) -> Option<&HelpOutput> { + self.help_output + .get_or_init(|| { + let runner = self.runner.as_ref()?; + HelpOutput::probe(runner).ok() + }) + .as_ref() + } + pub fn parsed_files(&self) -> &HashMap { self.parsed_files.get_or_init(|| { let mut cache = HashMap::new(); diff --git a/src/runner/help_probe.rs b/src/runner/help_probe.rs new file mode 100644 index 0000000..c6854fb --- /dev/null +++ b/src/runner/help_probe.rs @@ -0,0 +1,471 @@ +//! Shared `--help` probe + lazy parsers. +//! +//! The runner spawns ` --help` exactly once per target. The captured +//! text is parsed on demand into three views — flags, env hints, subcommands. +//! Behavioral checks that need to inspect the help surface share the same +//! `HelpOutput` for a given target so none of them re-spawn the binary. +//! +//! Parsers are English-only by convention: we match on clap's output shape +//! (`Commands:`, `[env: FOO]`, leading-whitespace flag lines). Localized help +//! is a named exception in `docs/coverage-matrix.md` — checks that consume +//! these parsers should Skip, not Warn, when the raw text lacks an English +//! help surface. + +use std::sync::OnceLock; + +use anyhow::Result; + +use super::{BinaryRunner, RunStatus}; + +/// A flag discovered in `--help` output. `short` is the single-character +/// variant (e.g., `-q`); `long` is the GNU-style variant (e.g., `--quiet`). +/// At least one of the two is always set. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Flag { + pub short: Option, + pub long: Option, +} + +impl Flag { + /// Whether this flag exposes `name` under either its short or long form. + /// Accepts `-s`, `--long`, or even `long` / `s` (without dashes). + pub fn matches(&self, name: &str) -> bool { + let with_dash_long = if name.starts_with('-') { + name.to_string() + } else if name.len() == 1 { + format!("-{name}") + } else { + format!("--{name}") + }; + self.short.as_deref() == Some(with_dash_long.as_str()) + || self.long.as_deref() == Some(with_dash_long.as_str()) + } +} + +/// A bound between a flag surface and an environment variable — surfaces +/// clap's `[env: FOO]` hints as first-class data so checks don't re-scan. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EnvHint { + /// Environment variable name, e.g., `RIPGREP_CONFIG_PATH`. + pub var: String, +} + +/// Shared, lazily-parsed view over ` --help`. Construct via +/// [`HelpOutput::probe`] in runner code, or [`HelpOutput::from_raw`] in tests. +pub struct HelpOutput { + raw: String, + flags: OnceLock>, + env_hints: OnceLock>, + /// Reserved for P3/P6 subcommand-structure checks. Parsed lazily like + /// the other views; no current behavioral check consumes it, so the + /// compiler would flag it as dead code without this allow. + #[allow(dead_code)] + subcommands: OnceLock>, +} + +impl HelpOutput { + /// Build a `HelpOutput` from captured help text. The primary seam for + /// unit tests — pass a fixture string and exercise the parsers without + /// spawning a binary. + pub fn from_raw(raw: impl Into) -> Self { + Self { + raw: raw.into(), + flags: OnceLock::new(), + env_hints: OnceLock::new(), + subcommands: OnceLock::new(), + } + } + + /// Spawn ` --help` via the shared `BinaryRunner` and capture its + /// combined stdout+stderr. Returns an empty `HelpOutput` rather than an + /// error on timeout/crash — a misbehaving `--help` is a signal the check + /// consumers can use, not a hard runner failure. + pub fn probe(runner: &BinaryRunner) -> Result { + let help = runner.run(&["--help"], &[]); + match help.status { + RunStatus::NotFound => { + anyhow::bail!("binary not found when probing --help") + } + RunStatus::PermissionDenied => { + anyhow::bail!("permission denied when probing --help") + } + RunStatus::Error(ref msg) => anyhow::bail!("--help probe failed: {msg}"), + // Ok / Timeout / Crash — capture whatever output is available. + // Some tools print help to stderr, or crash after writing usage. + RunStatus::Ok | RunStatus::Timeout | RunStatus::Crash { .. } => { + let mut raw = String::with_capacity(help.stdout.len() + help.stderr.len()); + raw.push_str(&help.stdout); + raw.push_str(&help.stderr); + Ok(Self::from_raw(raw)) + } + } + } + + /// Raw help text, exactly as captured. + pub fn raw(&self) -> &str { + &self.raw + } + + /// Flags parsed out of the help surface. Lazy + cached on first call. + pub fn flags(&self) -> &[Flag] { + self.flags.get_or_init(|| parse_flags(&self.raw)) + } + + /// `[env: FOO]` hints parsed out of the help surface. Lazy + cached. + pub fn env_hints(&self) -> &[EnvHint] { + self.env_hints.get_or_init(|| parse_env_hints(&self.raw)) + } + + /// Subcommand names parsed out of the help surface. Lazy + cached. + /// Reserved for P3/P6 checks; no behavioral check consumes this yet. + #[allow(dead_code)] + pub fn subcommands(&self) -> &[String] { + self.subcommands + .get_or_init(|| parse_subcommands(&self.raw)) + } +} + +/// Parse flag declarations from clap-style help text. +/// +/// A "flag line" is a line that starts with whitespace and then a dash. The +/// header portion (before the description) is split from the description by +/// two or more spaces — clap's canonical shape. We tokenize the header on +/// commas and whitespace, then classify each token as short (`-s`) or long +/// (`--long`). +fn parse_flags(raw: &str) -> Vec { + let mut flags = Vec::new(); + for line in raw.lines() { + if !line.starts_with(' ') { + continue; + } + let trimmed = line.trim_start(); + if !trimmed.starts_with('-') { + continue; + } + // Separator / heading lines like `---` are not flags. + if trimmed.starts_with("---") { + continue; + } + // Header = everything before clap's two-space description gap. When + // there's no description on the same line the whole remainder is the + // header. + let header = trimmed.split(" ").next().unwrap_or(trimmed); + + let mut short: Option = None; + let mut long: Option = None; + for piece in header.split(',') { + let candidate = piece.split_whitespace().next().unwrap_or(piece.trim()); + if candidate.is_empty() { + continue; + } + if let Some(long_name) = parse_long_flag(candidate) { + long = Some(long_name); + } else if let Some(short_name) = parse_short_flag(candidate) { + short = Some(short_name); + } + } + if short.is_some() || long.is_some() { + flags.push(Flag { short, long }); + } + } + flags +} + +/// Extract a `--long` flag name from a token like `--long`, `--long=`, +/// `--long[=]`, or `--long `. Returns `None` when `candidate` is +/// not a long flag. +fn parse_long_flag(candidate: &str) -> Option { + if !candidate.starts_with("--") || candidate.len() <= 2 { + return None; + } + // Walk the name chars: letters, digits, dashes, underscores. + let end = candidate[2..] + .find(|c: char| !(c.is_ascii_alphanumeric() || c == '-' || c == '_')) + .map(|i| i + 2) + .unwrap_or(candidate.len()); + if end <= 2 { + return None; + } + Some(candidate[..end].to_string()) +} + +/// Extract a `-s` short flag from a token like `-s`, `-s`, or `-s,`. +fn parse_short_flag(candidate: &str) -> Option { + let bytes = candidate.as_bytes(); + if bytes.len() < 2 || bytes[0] != b'-' { + return None; + } + // Second char must be a flag character (letter, digit, or `?`). + let c = bytes[1] as char; + if c.is_ascii_alphanumeric() || c == '?' { + Some(format!("-{c}")) + } else { + None + } +} + +/// Parse clap's `[env: FOO_BAR]` or `[env: FOO_BAR=]` annotations. +/// Each occurrence becomes one `EnvHint` — duplicates are preserved so +/// callers can count occurrences if useful. +fn parse_env_hints(raw: &str) -> Vec { + const TAG: &str = "[env:"; + let mut hints = Vec::new(); + let mut rest = raw; + while let Some(pos) = rest.find(TAG) { + let after = &rest[pos + TAG.len()..]; + let end = after.find(']').unwrap_or(after.len()); + let inner = after[..end].trim(); + let name = inner.split('=').next().unwrap_or("").trim(); + if is_env_var_name(name) { + hints.push(EnvHint { + var: name.to_string(), + }); + } + rest = &after[end..]; + } + hints +} + +/// Env var names are ASCII uppercase, digits, underscores; must start with +/// a letter or underscore. +fn is_env_var_name(s: &str) -> bool { + if s.is_empty() { + return false; + } + let first = s.as_bytes()[0] as char; + if !(first.is_ascii_uppercase() || first == '_') { + return false; + } + s.chars() + .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_') +} + +/// Parse the `Commands:` / `Subcommands:` block. We collect the first +/// whitespace-separated token on each line until the block terminates +/// (empty line, or a new non-indented section header). +#[allow(dead_code)] +fn parse_subcommands(raw: &str) -> Vec { + let mut out = Vec::new(); + let mut in_section = false; + for line in raw.lines() { + let trimmed = line.trim(); + let is_header = matches!(trimmed, "Commands:" | "Subcommands:" | "SUBCOMMANDS:"); + if is_header { + in_section = true; + continue; + } + if !in_section { + continue; + } + if trimmed.is_empty() { + // Blank line ends the block. + in_section = false; + continue; + } + if !line.starts_with(' ') { + // A new top-level section header ended the commands block. + break; + } + if let Some(name) = trimmed.split_whitespace().next() { + if is_subcommand_name(name) { + out.push(name.to_string()); + } + } + } + out +} + +/// Subcommand names are kebab-case/snake_case identifiers. Anything else — +/// `[options]`, ``, punctuation — is not a subcommand. +#[allow(dead_code)] +fn is_subcommand_name(s: &str) -> bool { + !s.is_empty() + && s.chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') + && s.chars().next().is_some_and(|c| c.is_ascii_alphanumeric()) +} + +#[cfg(test)] +mod tests { + use super::*; + + // A fixture modeled on ripgrep's `--help` — short+long flags, env hint. + const RIPGREP_HELP: &str = r#"ripgrep 14.1 + +Usage: rg [OPTIONS] PATTERN [PATH ...] + +Options: + -e, --regexp=PATTERN A pattern to search for. + --no-messages Suppress some error messages. + -q, --quiet Do not print anything to stdout. + -v, --invert-match Invert matching. + --null Print a NUL byte after file paths. + --color= When to use color. [env: RIPGREP_COLOR=] + --help Show this help message. + -V, --version Show version. +"#; + + // Modeled on clap's generated help, with a subcommand block and [env: ...]. + const CLAP_HELP: &str = r#"anc — the agent-native CLI linter + +Usage: anc + +Commands: + check Run checks against a CLI project or binary + completions Generate shell completions + generate Regenerate build artifacts + help Print this message or the help of the given subcommand + +Options: + -q, --quiet Suppress non-essential output [env: AGENTNATIVE_QUIET=] + -h, --help Print help + -V, --version Print version +"#; + + // Tool with no flags and no subcommands — env_hints parser must return empty. + const BARE_HELP: &str = r#"xurl-rs 0.1 +A tiny HTTP client. + +Usage: xurl-rs URL +"#; + + // Localized help — ensures parsers degrade to empty without panicking. + const NON_ENGLISH_HELP: &str = r#"用法: outil [选项] + +参数: + URL 目标网址 + +选项: + -H, --header
自定义请求头 + -X, --request HTTP 方法 +"#; + + #[test] + fn parse_flags_extracts_short_and_long() { + let flags = parse_flags(RIPGREP_HELP); + assert!(flags.iter().any(|f| f.short.as_deref() == Some("-q"))); + assert!(flags.iter().any(|f| f.long.as_deref() == Some("--quiet"))); + assert!( + flags + .iter() + .any(|f| f.long.as_deref() == Some("--no-messages")) + ); + assert!(flags.iter().any(|f| f.long.as_deref() == Some("--null"))); + } + + #[test] + fn parse_flags_handles_equals_and_values() { + let flags = parse_flags(RIPGREP_HELP); + // --regexp=PATTERN — the value shape must not leak into the long name. + let regexp = flags + .iter() + .find(|f| f.long.as_deref() == Some("--regexp")) + .expect("regexp flag parsed"); + assert_eq!(regexp.short.as_deref(), Some("-e")); + } + + #[test] + fn parse_flags_ignores_prose_dashes() { + // A line starting with '---' (separator) must not become a flag. + let src = "Usage: foo [OPTIONS]\n\n-------\n\nOptions:\n -q, --quiet Quiet mode.\n"; + let flags = parse_flags(src); + assert_eq!(flags.len(), 1); + assert_eq!(flags[0].short.as_deref(), Some("-q")); + } + + #[test] + fn parse_env_hints_captures_clap_style() { + let hints = parse_env_hints(RIPGREP_HELP); + assert!(hints.iter().any(|h| h.var == "RIPGREP_COLOR")); + } + + #[test] + fn parse_env_hints_multiple_occurrences() { + let hints = parse_env_hints(CLAP_HELP); + assert!(hints.iter().any(|h| h.var == "AGENTNATIVE_QUIET")); + } + + #[test] + fn parse_env_hints_rejects_invalid_names() { + // `[env: lowercase]` or `[env: 1ABC]` must not parse as env hints. + let src = " --flag [env: lowercase] [env: 1ABC] [env: VALID_1]"; + let hints = parse_env_hints(src); + assert_eq!(hints.len(), 1); + assert_eq!(hints[0].var, "VALID_1"); + } + + #[test] + fn parse_subcommands_reads_commands_block() { + let subs = parse_subcommands(CLAP_HELP); + assert!(subs.iter().any(|s| s == "check")); + assert!(subs.iter().any(|s| s == "generate")); + assert!(subs.iter().any(|s| s == "completions")); + } + + #[test] + fn parse_subcommands_empty_without_block() { + let subs = parse_subcommands(BARE_HELP); + assert!(subs.is_empty()); + } + + #[test] + fn parse_non_english_help_degrades_cleanly() { + // English-only parsers: no flags advertised via English conventions, + // no `Commands:` header, no `[env: ...]` hint — all parsers return empty. + let flags = parse_flags(NON_ENGLISH_HELP); + // The Chinese options block still uses `-H, --header` syntax so we may + // detect the flags themselves — the non-English text is in the + // descriptions, not the flag names. The check is that parsing doesn't + // panic and returns sane structured data. + for f in &flags { + assert!(f.short.is_some() || f.long.is_some()); + } + assert!(parse_env_hints(NON_ENGLISH_HELP).is_empty()); + assert!(parse_subcommands(NON_ENGLISH_HELP).is_empty()); + } + + #[test] + fn help_output_lazy_parse_is_idempotent() { + let help = HelpOutput::from_raw(RIPGREP_HELP); + // Pointer identity through two calls proves OnceLock caching. + let first = help.flags().as_ptr(); + let second = help.flags().as_ptr(); + assert_eq!(first, second); + // And the data is stable across calls. + assert_eq!(help.flags().len(), help.flags().len()); + } + + #[test] + fn flag_matches_accepts_various_spellings() { + let f = Flag { + short: Some("-q".into()), + long: Some("--quiet".into()), + }; + assert!(f.matches("-q")); + assert!(f.matches("--quiet")); + assert!(f.matches("quiet")); + assert!(f.matches("q")); + assert!(!f.matches("--verbose")); + } + + #[test] + fn is_env_var_name_edges() { + assert!(is_env_var_name("FOO")); + assert!(is_env_var_name("FOO_BAR")); + assert!(is_env_var_name("_UNDERSCORE")); + assert!(!is_env_var_name("")); + assert!(!is_env_var_name("lower")); + assert!(!is_env_var_name("1LEADING")); + assert!(!is_env_var_name("foo-bar")); + } + + #[test] + fn parse_short_flag_accepts_digits_and_question() { + assert_eq!(parse_short_flag("-q"), Some("-q".into())); + assert_eq!(parse_short_flag("-1"), Some("-1".into())); + assert_eq!(parse_short_flag("-?"), Some("-?".into())); + assert_eq!(parse_short_flag("--long"), None); + assert_eq!(parse_short_flag("-"), None); + assert_eq!(parse_short_flag("-,"), None); + } +} diff --git a/src/runner.rs b/src/runner/mod.rs similarity index 99% rename from src/runner.rs rename to src/runner/mod.rs index c4f1f7c..91d5498 100644 --- a/src/runner.rs +++ b/src/runner/mod.rs @@ -1,3 +1,7 @@ +pub mod help_probe; + +pub use help_probe::HelpOutput; + use std::cell::RefCell; use std::collections::HashMap; use std::io::Read as _; diff --git a/src/scorecard.rs b/src/scorecard.rs index e3e02c5..7c731f3 100644 --- a/src/scorecard.rs +++ b/src/scorecard.rs @@ -61,6 +61,9 @@ pub struct CheckResultView { pub layer: String, pub status: String, pub evidence: Option, + /// `high` for direct probes, `medium` for heuristics. Additive field; + /// v1.1 consumers feature-detect and tolerate missing keys. + pub confidence: String, } impl CheckResultView { @@ -72,7 +75,8 @@ impl CheckResultView { CheckStatus::Skip(e) => ("skip".to_string(), Some(e.clone())), CheckStatus::Error(e) => ("error".to_string(), Some(e.clone())), }; - // Serialize CheckGroup via serde_json for canonical format + // Serialize CheckGroup / CheckLayer / Confidence via serde_json so + // the JSON mirrors the canonical enum spelling (snake_case). let group = serde_json::to_value(r.group) .ok() .and_then(|v| v.as_str().map(|s| s.to_string())) @@ -81,6 +85,10 @@ impl CheckResultView { .ok() .and_then(|v| v.as_str().map(|s| s.to_string())) .unwrap_or_else(|| format!("{:?}", r.layer)); + let confidence = serde_json::to_value(r.confidence) + .ok() + .and_then(|v| v.as_str().map(|s| s.to_string())) + .unwrap_or_else(|| format!("{:?}", r.confidence)); CheckResultView { id: r.id.clone(), label: r.label.clone(), @@ -88,6 +96,7 @@ impl CheckResultView { layer, status, evidence, + confidence, } } } @@ -297,7 +306,7 @@ pub fn exit_code(results: &[CheckResult]) -> i32 { #[cfg(test)] mod tests { use super::*; - use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus}; + use crate::types::{CheckGroup, CheckLayer, CheckResult, CheckStatus, Confidence}; fn make_result(id: &str, status: CheckStatus, group: CheckGroup) -> CheckResult { CheckResult { @@ -306,6 +315,7 @@ mod tests { group, layer: CheckLayer::Behavioral, status, + confidence: Confidence::High, } } @@ -323,8 +333,10 @@ mod tests { assert_eq!(parsed["summary"]["fail"], 1); assert_eq!(parsed["results"][0]["status"], "pass"); assert!(parsed["results"][0]["evidence"].is_null()); + assert_eq!(parsed["results"][0]["confidence"], "high"); assert_eq!(parsed["results"][1]["status"], "fail"); assert_eq!(parsed["results"][1]["evidence"], "bad"); + assert_eq!(parsed["results"][1]["confidence"], "high"); // v1.1 additions: coverage_summary present with three levels, audience + audit_profile null. assert!(parsed["coverage_summary"]["must"]["total"].is_number()); assert!(parsed["coverage_summary"]["should"]["total"].is_number()); @@ -333,6 +345,14 @@ mod tests { assert!(parsed["audit_profile"].is_null()); } + #[test] + fn medium_confidence_serializes_as_medium() { + let mut r = make_result("c3", CheckStatus::Warn("soft".into()), CheckGroup::P6); + r.confidence = Confidence::Medium; + let view = CheckResultView::from_result(&r); + assert_eq!(view.confidence, "medium"); + } + #[test] fn coverage_summary_counts_verified_requirements() { use crate::check::Check; diff --git a/src/types.rs b/src/types.rs index 9161465..ba43749 100644 --- a/src/types.rs +++ b/src/types.rs @@ -12,6 +12,20 @@ pub enum CheckStatus { Error(String), } +/// How confident a check is in its verdict. Direct probes (flag parsers, +/// exit-code observation) report `High`; heuristic text inference reports +/// `Medium`; soft cross-signal inference reports `Low`. Consumers use this +/// to weight conflicting signals and surface caveats on the scorecard. +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq, Default)] +#[serde(rename_all = "snake_case")] +pub enum Confidence { + #[default] + High, + Medium, + #[allow(dead_code)] // Reserved for future inferential checks. + Low, +} + /// Groups checks by principle or category. #[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] #[allow(dead_code)] @@ -45,6 +59,10 @@ pub struct CheckResult { pub group: CheckGroup, pub layer: CheckLayer, pub status: CheckStatus, + /// How much the check trusts its own verdict. Defaults to `High`; only + /// heuristic checks downgrade. Additive field; consumers feature-detect. + #[serde(default)] + pub confidence: Confidence, } /// A source location where a violation was found.