From 8c91c1673f6c2cf8835df72b047a8eaa015d1c66 Mon Sep 17 00:00:00 2001 From: Haksung Jang Date: Mon, 24 Aug 2026 15:04:53 +0900 Subject: [PATCH] feat(scans): skip cdxgen when a commit's dependency set is unchanged (S8) Fingerprints a scan's manifest/lockfile set, cdxgen scanner version, and scan-time config; when it matches the project's prior succeeded scan on the same ref, the pipeline reuses that scan's preserved SBOM instead of re-running cdxgen (5-30 min), while vulnerability matching and license classification always re-run against current data. A scanner-version bump or config change invalidates the fingerprint automatically. Schema and the pure fingerprint function were built in a prior session; this adds the reuse decision, the extraction/fallback wiring in tasks/scan_source.py, and the cdxgen_scanner_version() config accessor. --- .env.example | 8 + .../0070_scan_dependency_fingerprint.py | 83 ++++ apps/backend/core/config.py | 27 ++ apps/backend/models/scan.py | 31 ++ apps/backend/models/scan_fingerprint.py | 189 ++++++++ apps/backend/tasks/scan_source.py | 388 ++++++++++++--- ...can_source_dependency_fingerprint_reuse.py | 449 ++++++++++++++++++ ...t_scan_dependency_fingerprint_migration.py | 220 +++++++++ .../tests/unit/tasks/test_scan_chaos.py | 2 + ..._source_dependency_fingerprint_fallback.py | 303 ++++++++++++ .../tasks/test_scan_source_load_test_delay.py | 2 + .../tasks/test_scan_source_scope_filter.py | 47 +- .../tests/unit/tasks/test_scan_timeout.py | 3 + .../tests/unit/test_scan_fingerprint.py | 348 ++++++++++++++ 14 files changed, 2014 insertions(+), 86 deletions(-) create mode 100644 apps/backend/alembic/versions/0070_scan_dependency_fingerprint.py create mode 100644 apps/backend/models/scan_fingerprint.py create mode 100644 apps/backend/tests/integration/scan/test_scan_source_dependency_fingerprint_reuse.py create mode 100644 apps/backend/tests/integration/test_scan_dependency_fingerprint_migration.py create mode 100644 apps/backend/tests/unit/tasks/test_scan_source_dependency_fingerprint_fallback.py create mode 100644 apps/backend/tests/unit/test_scan_fingerprint.py diff --git a/.env.example b/.env.example index 565b2edd..e464866c 100644 --- a/.env.example +++ b/.env.example @@ -483,6 +483,14 @@ SCANOSS_TIMEOUT_SECONDS=300 # CDXGEN_SPEC_VERSION=1.5 # set 1.6 to emit CycloneDX 1.6 # CDXGEN_FETCH_LICENSE=false # true → cdxgen resolves component licenses (slower) # +# CDXGEN_VERSION=12.3.3 # set by Dockerfile.worker's ENV at image-build time, NOT +# # an operator-facing setting — do not set in .env. Names the +# # cdxgen release pinned in the worker image; folds into the +# # S8 dependency-set fingerprint (tasks/scan_source.py) so an +# # upgraded worker's first scan on an otherwise-unchanged tree +# # is never treated as reusable against a fingerprint an older +# # worker wrote. +# # Post-cdxgen license enrichment. When cdxgen emits a component with # no SPDX license — the common case for a bare requirements.txt / go.mod with # no installed packages — the pipeline asks the component's PUBLIC registry diff --git a/apps/backend/alembic/versions/0070_scan_dependency_fingerprint.py b/apps/backend/alembic/versions/0070_scan_dependency_fingerprint.py new file mode 100644 index 00000000..e85ca832 --- /dev/null +++ b/apps/backend/alembic/versions/0070_scan_dependency_fingerprint.py @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 TRUSCA contributors +"""scan dependency fingerprint + +Revision ID: 0070 +Revises: 0069 +Create Date: 2026-08-24 + +Phase: concurrency-scaling-plan-2026-08-22.md §3.2 S8 (unit 29), schema step +PR: (opened together with the follow-on reuse-decision revision) +Kind: schema (additive, one nullable column; no data migration) +Forward-only: yes + +What: + ``scans.dependency_fingerprint``: a SHA-256 hex digest (64 chars, fixed + length) over the scan's manifest/lockfile hashes, the cdxgen scanner + version, and the scan-time config that shapes the generated SBOM. Nullable + ``VARCHAR(64)``, no default. + +Why: + S8 skips re-running cdxgen when a commit's dependency set has not changed + since the project's last successful scan, reusing that scan's preserved + SBOM for vulnerability re-matching instead (the reuse machinery already + exists: ``tasks.vulnerability_rematch.preserved_tarball_has_sbom`` + + ``rematch_scan_findings``). Deciding "has not changed" needs something to + compare against, and nothing on the ``scans`` row currently records what a + scan's dependency set actually was. This migration adds the one column the + comparison needs; the fingerprint is computed and written by + ``models.scan_fingerprint.compute_scan_fingerprint`` at scan-success time + (wired into the pipeline in a follow-on change, see the plan's §7.1 unit + 29 "머지 단위: 연속"). + + This is a schema-only step by design (plan §8: "스키마를 건드리는 단위 ... + forward-only 마이그레이션이고 다운그레이드를 두지 않는다"). The column is + added and populated going forward; no backfill is attempted for scans that + already succeeded, because their preserved source tree is gone by the time + this migration runs for most of them (retained latest-succeeded-per-project + only, see migration 0051's rationale) and a fingerprint computed from + nothing would be indistinguishable from one computed from a since-deleted + tree that no longer matches. NULL is deliberately not a value the reuse + decision (a later revision) will treat as a match. + +Why a column and not a table: + One scalar per scan, read for exactly one comparison ("does this scan's + fingerprint match the prior succeeded scan's for the same (project_id, + ref)?"), never filtered or aggregated across scans. A satellite table would + need its own FK, index, and cascade-delete wiring to answer a question a + single column already answers. + +Index: + None. The lookup this column supports is "the latest succeeded scan for + this (project_id, ref)", already served by the existing partial index + ``ix_scans_project_ref`` (status = 'succeeded'). The reuse decision reads + that row's ``dependency_fingerprint`` scalar directly; it does not search + BY fingerprint, so no new index earns its write cost. + +Reversal: + Forward-only, and additive. With the column unread by anything outside the + (not-yet-wired) reuse decision, every existing code path is unaffected. +""" + +from __future__ import annotations + +import sqlalchemy as sa + +from alembic import op + +revision: str = "0070" +down_revision: str | None = "0069" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + op.add_column( + "scans", + sa.Column("dependency_fingerprint", sa.String(length=64), nullable=True), + ) + + +def downgrade() -> None: + # Forward-only (CLAUDE.md §6). + raise NotImplementedError("0070 is forward-only") diff --git a/apps/backend/core/config.py b/apps/backend/core/config.py index b83f7ec7..72c342f5 100644 --- a/apps/backend/core/config.py +++ b/apps/backend/core/config.py @@ -1039,6 +1039,33 @@ def cdxgen_fetch_license() -> bool: } +def cdxgen_scanner_version() -> str: + """The cdxgen version baked into this worker image (``CDXGEN_VERSION``). + + S8 (concurrency-scaling-plan-2026-08-22.md §3.2): the scan pipeline + fingerprints its dependency set to decide whether a fresh commit can + reuse a prior scan's SBOM instead of re-running cdxgen. Two trees with + byte-identical manifests can still produce different SBOMs after a + cdxgen upgrade, so the fingerprint must fold in the scanner version (an + upgraded worker's first scan on an otherwise-unchanged tree must never + read as "unchanged" against a fingerprint an older worker wrote). + + ``Dockerfile.worker`` sets ``ENV CDXGEN_VERSION=`` at + image-build time (see the ``cdxgen`` install stage there); this accessor + reads that at call time (rule #11) rather than caching it, so a value + baked into one image build is never carried over by a stale in-process + cache after a hot-swap. Returns ``"unknown"`` when unset, the same + non-placeholder convention :func:`slsa_builder_version` uses for + ``TRUSTEDOSS_VERSION``, which still participates in the fingerprint + hash (an "unknown"-tagged scan's fingerprint is simply never treated as + matching a version-tagged one unless both literally say "unknown"). + """ + raw = os.getenv("CDXGEN_VERSION") + if raw is None or raw.strip() == "": + return "unknown" + return raw.strip() + + def license_fetch_enabled() -> bool: """Whether the post-cdxgen license fetcher enriches unlicensed components. diff --git a/apps/backend/models/scan.py b/apps/backend/models/scan.py index 1c18d3ea..273a759b 100644 --- a/apps/backend/models/scan.py +++ b/apps/backend/models/scan.py @@ -386,6 +386,37 @@ class Scan(Base): input_document: Mapped[dict[str, Any] | None] = mapped_column( JSONB, nullable=True ) + # Dependency-set fingerprint (S8, concurrency-scaling-plan-2026-08-22.md + # §3.2, migration 0070). SHA-256 hex digest over three inputs that + # together determine the bytes of the SBOM this scan would generate: the + # scanned tree's manifest/lockfile hashes (the same inventory recorded in + # ``input_manifests`` above; see ``services.scan_inputs. + # collect_manifest_inventory``), the cdxgen scanner version, and the + # scan-time config that shapes cdxgen's output (spec version, + # license-fetch toggle, runtime-scope filter toggles). Computed by + # ``models.scan_fingerprint.compute_scan_fingerprint``. + # + # NULL means "not computed": every scan before this migration, every + # container / SBOM-ingest scan (neither has a source tree to + # fingerprint), and any scan whose manifest walk was truncated or could + # not hash one of its own files in full (the same bounds + # ``collect_manifest_inventory`` already enforces). A NULL fingerprint is + # never equal to another NULL: two scans that were not fingerprinted have + # not been shown to share a dependency set, they simply were not compared. + # + # Deliberately excludes vulnerability-DB state and license policy: both + # are inputs to the vulnerability-matching stage, not to SBOM generation, + # and the reuse decision this fingerprint exists to support (a later + # revision) always re-runs matching regardless of whether the SBOM itself + # is reused; see the plan's "판단" paragraph under S8. + # + # Index: none. The natural lookup is "the latest succeeded scan for this + # (project_id, ref)", already served by ``ix_scans_project_ref`` below; + # the caller reads that row's ``dependency_fingerprint`` scalar and + # compares in application code rather than searching BY fingerprint. + dependency_fingerprint: Mapped[str | None] = mapped_column( + String(64), nullable=True + ) # DT-style ref-keyed retention (scan-retention). Normalized git ref this # scan targets — ``refs/heads/main`` → ``main``, ``refs/pull/12/merge`` → # ``pr-12`` (see ``services.scan_service.normalize_ref``). NULL when the diff --git a/apps/backend/models/scan_fingerprint.py b/apps/backend/models/scan_fingerprint.py new file mode 100644 index 00000000..3bbcadad --- /dev/null +++ b/apps/backend/models/scan_fingerprint.py @@ -0,0 +1,189 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 TRUSCA contributors +""" +Dependency-set fingerprint for ``scans.dependency_fingerprint``. + +S8 (concurrency-scaling-plan-2026-08-22.md §3.2) skips re-running cdxgen when +a commit's dependency set has not changed since the project's last successful +scan on the same ref, reusing that scan's preserved SBOM for vulnerability +re-matching instead. This module computes the fingerprint that comparison +needs. It does NOT decide whether to reuse anything: that decision (reading +two fingerprints and choosing a pipeline path) belongs to a later revision +that also has to make that choice at a point in the pipeline this module has +no visibility into. This module is one pure function: given the same +inputs, always the same digest. + +What must be true of the inputs, and why: + + Lockfile / manifest hashes decide the dependency SET. This deliberately + reuses ``services.scan_inputs.collect_manifest_inventory`` (the same + per-ecosystem catalog of manifest AND lockfile names already gathered for + scan provenance, ``scans.input_manifests``) rather than a narrower + "lockfiles only" list. Several ecosystems this product scans have no + separate lockfile at all (Maven's ``pom.xml``, a bare ``requirements.txt``, + Gradle without dependency locking): for those, the manifest IS the + authoritative declaration of what gets resolved, and a fingerprint that + ignored it would call an SBOM reusable after a dependency version changed + in exactly those ecosystems. Where a lockfile does exist (``package-lock. + json``, ``poetry.lock``, ``go.sum``, ...) it is already in the same + catalog, so nothing is lost by not special-casing it. + + Scanner version and scan config decide what cdxgen does WITH that + dependency set. Two scans that saw byte-identical lockfiles can still + produce different SBOMs if the scanner was upgraded or a toggle that + shapes its output changed in between; the plan calls this out explicitly + as the accuracy requirement S8 must not violate. Both are supplied by the + caller (this module has no access to ``core.config`` or the filesystem by + design, see "Division of responsibility" below), so the pipeline is the + single place that decides which config keys count as "shapes SBOM output" + and stays free to add one without this module changing. + + Vulnerability-DB state and license policy are NOT inputs here on purpose. + Both are inputs to the vulnerability-MATCHING stage, not to SBOM + generation, and the reuse design this fingerprint exists to support always + re-runs matching regardless of whether the SBOM itself is reused (plan + §3.2: "둘 다 SBOM이 아니라 매칭 단계의 입력이므로 매칭을 다시 도는 설계와 + 어긋나지 않는다"). + +Division of responsibility (why this lives in ``models/`` and stays pure): + The obvious home for a hash-computation helper in this codebase is + ``services/`` (see ``services.remediation_pr_service``'s + ``change_fingerprint``, the closest existing precedent). This module is + colocated with the ``Scan`` model instead, and takes its scanner-version + and scan-config inputs as plain arguments rather than reading + ``core.config`` or invoking cdxgen itself, because computing and WRITING + the fingerprint at scan-success time (the part that needs + ``core.config.cdxgen_spec_version`` and friends, and a place in + ``tasks.scan_source``'s pipeline) is a follow-on change outside this + session's scope. Keeping this function pure and dependency-free means that + follow-on change can import it from wherever it ends up living without + this module needing to move first. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping, Sequence +from typing import Any, Final + +#: Bumped whenever the hashing shape changes: which fields are hashed, in +#: what order, or how they are serialized. A code change that touches this +#: module without also changing any lockfile byte, scanner version, or scan +#: config must still produce a fingerprint that differs from what an older +#: worker would have written for the identical scan; otherwise the very +#: first scan a new worker processes after an unrelated fingerprint-format +#: change could read as "unchanged" against a fingerprint an old worker +#: wrote under a different, incompatible scheme. +FINGERPRINT_SCHEMA_VERSION: Final = 1 + +#: JSON-serializable scalar types a scan-config value may hold. Anything else +#: is coerced to ``str`` (see ``_normalize_config``) rather than rejected; +#: this function must never raise on a config shape it was not expecting, +#: the same defensive posture ``services.scan_inputs`` takes throughout. +_ConfigScalar = bool | int | float | str + + +def compute_scan_fingerprint( + *, + manifest_inventory: Mapping[str, Any] | None, + scanner_version: str, + scan_config: Mapping[str, Any], +) -> str | None: + """Return a deterministic SHA-256 hex digest, or None when one cannot be trusted. + + ``manifest_inventory`` is the exact shape + ``services.scan_inputs.collect_manifest_inventory`` returns (and + ``scans.input_manifests`` stores): ``{"files": [{"path", "size", + "sha256"}, ...], "count", "truncated"}``. Passing the already-collected + inventory instead of a source directory keeps this function filesystem- + free: the walk, its bounds, and its skip-list live in exactly one place + (``scan_inputs.py``) and this function trusts that place's judgment about + what counts as a dependency declaration. + + Returns ``None`` (deliberately, not a digest of "nothing") when the + inventory cannot be trusted to describe the WHOLE tree: + + - ``manifest_inventory`` is ``None``: no manifest/lockfile was found + (or the scan has no source tree, e.g. container / SBOM-ingest scans). + There is no dependency-set identity to fingerprint. + - ``manifest_inventory["truncated"]`` is true: the walk stopped at + ``scan_inputs.MAX_ENTRIES`` before covering the tree. A file that + changed past the cutoff would go undetected, which is exactly the + failure mode a fingerprint exists to prevent. + - any recorded file has ``sha256`` of ``None``: that file was too large + to hash (``scan_inputs.MAX_HASH_BYTES``) or could not be read. + "Unknown content" and "unchanged content" must never collide. + + A ``None`` return must never be treated as equal to another ``None`` by + a caller comparing two scans' fingerprints: two un-fingerprinted scans + have not been shown to share a dependency set, they simply were not + compared. Callers enforce that; this function only refuses to assert a + digest it cannot stand behind. + + Deterministic: two calls with equal arguments (independent of Python + dict insertion order; every mapping is sorted before serializing) + always return the same digest. Two calls where any manifest hash, the + scanner version, or any scan-config value differs return different + digests with overwhelming probability (SHA-256 preimage resistance). + """ + if not manifest_inventory: + return None + if manifest_inventory.get("truncated"): + return None + + files = manifest_inventory.get("files") + if not isinstance(files, Sequence) or isinstance(files, str | bytes) or not files: + return None + + entries: list[tuple[str, str]] = [] + for entry in files: + if not isinstance(entry, Mapping): + return None + path = entry.get("path") + digest = entry.get("sha256") + if not isinstance(path, str) or not path: + return None + if not isinstance(digest, str) or not digest: + # Too large to hash, or unreadable this pass (scan_inputs._sha256 + # returns None in both cases). "Unchanged" cannot be claimed + # about a file whose content this scan never actually read. + return None + entries.append((path, digest)) + + # Sort defensively even though collect_manifest_inventory already sorts + # its output: this function's determinism must not depend on a caller + # upholding an invariant it cannot verify from the shape alone. + entries.sort(key=lambda pair: pair[0]) + + payload = { + "schema": FINGERPRINT_SCHEMA_VERSION, + "files": entries, + "scanner_version": scanner_version, + "scan_config": _normalize_config(scan_config), + } + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _normalize_config(scan_config: Mapping[str, Any]) -> dict[str, Any]: + """Coerce scan-config values to JSON-serializable scalars. + + A config value this function was not told to expect (a nested dict, a + custom object) is stringified rather than dropped or raising: dropping + it would silently narrow the fingerprint's inputs, and this function has + no basis to decide such a value is safe to ignore. + """ + normalized: dict[str, Any] = {} + for key, value in scan_config.items(): + if value is None or isinstance(value, bool | int | float | str): + normalized[key] = value + else: + normalized[key] = str(value) + return normalized + + +__all__ = [ + "FINGERPRINT_SCHEMA_VERSION", + "compute_scan_fingerprint", +] diff --git a/apps/backend/tasks/scan_source.py b/apps/backend/tasks/scan_source.py index e3164881..40e06daf 100644 --- a/apps/backend/tasks/scan_source.py +++ b/apps/backend/tasks/scan_source.py @@ -74,6 +74,7 @@ from core.config import ( cdxgen_fetch_license, + cdxgen_scanner_version, cdxgen_spec_version, eol_enabled, license_fetch_enabled, @@ -125,6 +126,7 @@ ScanComponent, VulnerabilityFinding, ) +from models.scan_fingerprint import compute_scan_fingerprint from services import sbom_component_walk, sbom_document_metadata, scan_inputs from services.component_approval_service import ( apply_intake_decisions, @@ -141,7 +143,13 @@ resolve_existing_archive, safe_extract_archive, ) -from services.source_preservation_service import preserve_scan_source +from services.source_preservation_service import ( + PreservationTooLarge, + PreservedSbomMissing, + SourcePreservationError, + extract_preserved_sbom, + preserve_scan_source, +) from services.vulnerability_matching import persist_trivy_findings from tasks._progress import ( close_log_file, @@ -263,6 +271,11 @@ def scan_source_task(self: Any, scan_id: str) -> None: # async engine. A plain dict copy is safe to carry into the # pipeline. scan_metadata = dict(scan.scan_metadata or {}) + # S8 (concurrency-scaling-plan-2026-08-22.md §3.2): the reuse + # decision compares this scan against the prior succeeded scan + # for the SAME (project_id, ref), snapshotted here for the same + # session-detach reason as scan_metadata above. + scan_ref = scan.ref # M1/M2 (concurrency-scaling plan) load-test mode: hold this worker # slot busy for a fixed delay instead of running cdxgen/scancode/Trivy, @@ -283,6 +296,7 @@ def scan_source_task(self: Any, scan_id: str) -> None: workspace=workspace, git_url=project_git_url, scan_metadata=scan_metadata, + ref=scan_ref, ) except _FetchAborted as exc: # SSRF guard / fetch refused the project URL — terminal, not a @@ -350,6 +364,7 @@ def _run_pipeline( workspace: Path, git_url: str | None, scan_metadata: dict[str, Any] | None = None, + ref: str | None = None, ) -> None: """Execute the scan stages, each with its own commit.""" # Scan-log verbosity (feat/scan-log-verbosity): a per-scan @@ -394,81 +409,154 @@ def _run_pipeline( detected_env = _detect_and_record_env(scan_uuid, project_root) # Scan provenance (gap #31): what the fetched tree declared, recorded before - # prep writes any lockfile of its own. - _record_input_manifests(scan_uuid, project_root) - - # Stage 2.5 + 3 — build-prep + cdxgen, behind the ScanExecutor abstraction. - # cdxgen needs a populated lockfile to enumerate transitive deps for Ruby / - # Rust / Go / .NET; the 2026-05-07 ecosystem-matrix UAT showed bare-source - # scans returned 0 or only direct deps for those four ecosystems. The - # default in-process executor runs prep + cdxgen as worker-local subprocesses - # exactly as before (behaviour-preserving); SCAN_EXECUTOR=local_docker / - # k8s_job route environment-specific cdxgen sidecars instead (later - # increments). The in-process prep is INJECTED as a callable so this module - # stays the only importer of the executor package (no import cycle). - # - # prep is best-effort: a failed prep logs a warning and the scan continues - # with whatever cdxgen can extract (see _prepare_for_cdxgen). The executor - # advances "prep" then "cdxgen" so the percent/progress contract is intact. - executor = scan_executor.get_executor() - gen_request = scan_executor.SbomGenRequest( - scan_uuid=scan_uuid, - source_dir=source_dir, - output_dir=workspace / "cdxgen", - detected_env=detected_env, - # K-f2: carry the resolved project root so a container sidecar targets - # the same directory ``detected_env`` was detected from (a git clone - # lands under source_dir; non-recursive detection + the sidecar's - # single-dir scan would otherwise mis-target the outer dir). - project_root=project_root, - verbose=verbose, - # spec-version / fetch-license toggles, resolved per-scan (rule #11) and - # carried on the request so both the in-process and sidecar executors - # apply the same values. - spec_version=cdxgen_spec_version(), - fetch_license=cdxgen_fetch_license(), - ) - gen_result = executor.generate_sbom( - gen_request, - prep=lambda: _prepare_for_cdxgen(source_dir=project_root, scan_uuid=scan_uuid), - stage=lambda stage: _set_stage(scan_uuid, stage), - # P2 #8c — stream cdxgen stdout/stderr lines onto the scan WebSocket - # so the drawer can render a live tool trace. Best-effort: a publish - # error inside the callback never propagates, and the per-scan line - # budget caps runaway tools (publish_log enforces it internally). - line_callback=_make_line_callback(scan_uuid, stage="cdxgen"), + # prep writes any lockfile of its own. The returned inventory is also the + # S8 fingerprint's manifest input (below): read once, used twice, rather + # than a second directory walk. + manifest_inventory = _record_input_manifests(scan_uuid, project_root) + + # S8 (concurrency-scaling-plan-2026-08-22.md §3.2): decide, BEFORE cdxgen + # runs, whether this commit's dependency set is unchanged from the prior + # succeeded scan on the same (project, ref). The fingerprint folds in the + # scanner version and the scan-time cdxgen config so an upgraded worker or + # a changed toggle never reads as "unchanged" (accuracy requirement, plan + # §3.2 "주의할 것은 정확성이다"). Vulnerability-DB state and license policy + # are deliberately excluded: matching always re-runs regardless of reuse + # (models.scan_fingerprint module docstring). + dependency_fingerprint = compute_scan_fingerprint( + manifest_inventory=manifest_inventory, + scanner_version=cdxgen_scanner_version(), + scan_config={ + "cdxgen_spec_version": cdxgen_spec_version(), + "cdxgen_fetch_license": cdxgen_fetch_license(), + "scan_scope_filter_enabled": scan_scope_filter_enabled(), + "scan_scope_filter_maven_enabled": scan_scope_filter_maven_enabled(), + "scan_scope_filter_node_enabled": scan_scope_filter_node_enabled(), + }, ) - cdxgen_result = cdxgen_adapter.CdxgenResult( - sbom_path=gen_result.sbom_path, sbom=gen_result.sbom + reuse_source_scan_id = _find_reusable_prior_scan( + project_id=project_id, + ref=ref, + scan_uuid=scan_uuid, + fingerprint=dependency_fingerprint, ) - # Stage 3.2 — CocoaPods lockfile fill-in (Phase L). cdxgen ran with - # --exclude-type cocoapods when a Podfile was present (its cataloger - # crashes without the `pod` CLI — see integrations/cdxgen.py), so the - # pods are reconstructed offline from Podfile.lock and merged here. - # MUST precede the scope filter so the filter's counts and the - # trusca:scope_filter property describe the FINAL document (cocoapods - # purls pass both keep-predicates untouched). Best-effort, never fatal. - _merge_cocoapods_components( - scan_uuid=scan_uuid, cdxgen_result=cdxgen_result, source_dir=project_root - ) + # S8: try the reuse path first. A successful extraction gives us the + # PRIOR scan's fully-processed SBOM (cocoapods-merged, scope-filtered, + # metadata-stamped already, because that is exactly what + # ``_preserve_source_tree`` archived for that scan, below). None of those + # three stages need to (or should) run again over bytes that already + # carry their output. A failed extraction (tarball missing/corrupt since + # the fingerprint was written, e.g. reclaimed by retention) transparently + # falls back to the full cdxgen path, since reuse is an optimization, + # never a correctness dependency. + cdxgen_result: cdxgen_adapter.CdxgenResult | None = None + if reuse_source_scan_id is not None: + cdxgen_result = _reuse_prior_sbom( + scan_uuid=scan_uuid, + project_id=project_id, + prior_scan_id=reuse_source_scan_id, + workspace=workspace, + ) + if cdxgen_result is None: + reuse_source_scan_id = None + + if cdxgen_result is not None: + # Reused path: still advance through "prep" then "cdxgen" so a WS + # client watching this scan sees the identical monotonic stage + # sequence the full pipeline would have driven through the executor's + # own ``stage=`` callback (see InProcessExecutor.generate_sbom). The + # regression contract requires the reuse and full paths to be + # indistinguishable from the outside. + _set_stage(scan_uuid, "prep") + _set_stage(scan_uuid, "cdxgen") + log.info( + "scan_dependency_fingerprint_reused", + scan_id=str(scan_uuid), + prior_scan_id=str(reuse_source_scan_id), + ) + else: + # Stage 2.5 + 3: build-prep + cdxgen, behind the ScanExecutor + # abstraction. cdxgen needs a populated lockfile to enumerate + # transitive deps for Ruby / Rust / Go / .NET; the 2026-05-07 + # ecosystem-matrix UAT showed bare-source scans returned 0 or only + # direct deps for those four ecosystems. The default in-process + # executor runs prep + cdxgen as worker-local subprocesses exactly as + # before (behaviour-preserving); SCAN_EXECUTOR=local_docker / k8s_job + # route environment-specific cdxgen sidecars instead (later + # increments). The in-process prep is INJECTED as a callable so this + # module stays the only importer of the executor package (no import + # cycle). + # + # prep is best-effort: a failed prep logs a warning and the scan + # continues with whatever cdxgen can extract (see + # _prepare_for_cdxgen). The executor advances "prep" then "cdxgen" so + # the percent/progress contract is intact. + executor = scan_executor.get_executor() + gen_request = scan_executor.SbomGenRequest( + scan_uuid=scan_uuid, + source_dir=source_dir, + output_dir=workspace / "cdxgen", + detected_env=detected_env, + # K-f2: carry the resolved project root so a container sidecar + # targets the same directory ``detected_env`` was detected from + # (a git clone lands under source_dir; non-recursive detection + + # the sidecar's single-dir scan would otherwise mis-target the + # outer dir). + project_root=project_root, + verbose=verbose, + # spec-version / fetch-license toggles, resolved per-scan + # (rule #11) and carried on the request so both the in-process + # and sidecar executors apply the same values. + spec_version=cdxgen_spec_version(), + fetch_license=cdxgen_fetch_license(), + ) + gen_result = executor.generate_sbom( + gen_request, + prep=lambda: _prepare_for_cdxgen( + source_dir=project_root, scan_uuid=scan_uuid + ), + stage=lambda stage: _set_stage(scan_uuid, stage), + # P2 #8c: stream cdxgen stdout/stderr lines onto the scan + # WebSocket so the drawer can render a live tool trace. + # Best-effort: a publish error inside the callback never + # propagates, and the per-scan line budget caps runaway tools + # (publish_log enforces it internally). + line_callback=_make_line_callback(scan_uuid, stage="cdxgen"), + ) + cdxgen_result = cdxgen_adapter.CdxgenResult( + sbom_path=gen_result.sbom_path, sbom=gen_result.sbom + ) - # Stage 3.25 — runtime-scope post-filter (Phase K). MUST run before the - # artifact persist, cosign signing, SCANOSS and Trivy below so every - # downstream consumer (persisted components, the signed/downloadable - # artifact, vulnerability matching) sees ONE consistent filtered document. - # Best-effort: any failure leaves the SBOM exactly as cdxgen wrote it. - _apply_scope_filter( - scan_uuid=scan_uuid, cdxgen_result=cdxgen_result, source_dir=project_root - ) + # Stage 3.2: CocoaPods lockfile fill-in (Phase L). cdxgen ran with + # --exclude-type cocoapods when a Podfile was present (its cataloger + # crashes without the `pod` CLI (see integrations/cdxgen.py), so the + # pods are reconstructed offline from Podfile.lock and merged here. + # MUST precede the scope filter so the filter's counts and the + # trusca:scope_filter property describe the FINAL document (cocoapods + # purls pass both keep-predicates untouched). Best-effort, never + # fatal. + _merge_cocoapods_components( + scan_uuid=scan_uuid, cdxgen_result=cdxgen_result, source_dir=project_root + ) - # Stage 3.3 — document metadata (2026 SBOM minimum elements). MUST run - # after the scope filter and BEFORE the artifact persist and cosign - # signing, so the bytes we sign are the bytes that carry the statements. - # This is the SBOM a consumer actually receives; the export route - # (services/sbom_export) stamps the same things on its own document. - # Best-effort: a failure leaves the SBOM exactly as it was. - _stamp_document_metadata(scan_uuid=scan_uuid, cdxgen_result=cdxgen_result) + # Stage 3.25: runtime-scope post-filter (Phase K). MUST run before + # the artifact persist, cosign signing, SCANOSS and Trivy below so + # every downstream consumer (persisted components, the + # signed/downloadable artifact, vulnerability matching) sees ONE + # consistent filtered document. Best-effort: any failure leaves the + # SBOM exactly as cdxgen wrote it. + _apply_scope_filter( + scan_uuid=scan_uuid, cdxgen_result=cdxgen_result, source_dir=project_root + ) + + # Stage 3.3: document metadata (2026 SBOM minimum elements). MUST + # run after the scope filter and BEFORE the artifact persist and + # cosign signing, so the bytes we sign are the bytes that carry the + # statements. This is the SBOM a consumer actually receives; the + # export route (services/sbom_export) stamps the same things on its + # own document. Best-effort: a failure leaves the SBOM exactly as it + # was. + _stamp_document_metadata(scan_uuid=scan_uuid, cdxgen_result=cdxgen_result) _persist_artifact( scan_uuid, @@ -694,6 +782,17 @@ def _run_pipeline( sbom_path=cdxgen_result.sbom_path, ) + # S8: stamp the fingerprint computed above onto the scan row NOW. A scan + # that does not reach this point (any terminal failure) keeps + # ``dependency_fingerprint`` NULL, and NULL is never treated as a reuse + # match by ``_find_reusable_prior_scan`` above. Written unconditionally of + # whether THIS scan itself took the reuse path: a reused scan's SBOM is + # byte-identical to its source scan's, so its fingerprint is the correct + # description of what it now offers as a reuse candidate for the NEXT + # scan on this ref, extending the reuse chain instead of collapsing it + # back to a two-scan cycle. + _persist_dependency_fingerprint(scan_uuid, dependency_fingerprint) + # Stage 7 — finalize. _set_stage(scan_uuid, "finalize") _mark_succeeded(scan_uuid) @@ -1446,7 +1545,9 @@ def _detect_and_record_env(scan_uuid: uuid.UUID, source_dir: Path) -> str: return detected_env -def _record_input_manifests(scan_uuid: uuid.UUID, project_root: Path) -> None: +def _record_input_manifests( + scan_uuid: uuid.UUID, project_root: Path +) -> dict[str, Any] | None: """Record the dependency manifests the fetched tree carried (gap #31). Runs BEFORE build-prep, deliberately. Prep generates lockfiles for the @@ -1458,10 +1559,15 @@ def _record_input_manifests(scan_uuid: uuid.UUID, project_root: Path) -> None: Best-effort like every other observation on this path: a failure logs a warning and leaves the column NULL, which reads as "not recorded" rather than "nothing found" (migration 0051). + + Returns the collected inventory (the same shape written to + ``scan.input_manifests``, ``None`` when nothing was found or persistence + failed) so S8's fingerprint computation can reuse it without walking the + tree a second time. """ inventory = scan_inputs.collect_manifest_inventory(project_root) if inventory is None: - return + return None log.info( "scan_input_manifests", @@ -1481,6 +1587,142 @@ def _record_input_manifests(scan_uuid: uuid.UUID, project_root: Path) -> None: scan_id=str(scan_uuid), exc_info=True, ) + return None + return inventory + + +def _persist_dependency_fingerprint( + scan_uuid: uuid.UUID, fingerprint: str | None +) -> None: + """S8: write this scan's dependency-set fingerprint at success time. + + Called once, right before ``_mark_succeeded``. Best-effort like + ``_record_input_manifests`` above: a persistence failure logs a WARNING + and leaves the column NULL rather than failing an otherwise-successful + scan. The worst case is that the NEXT scan on this ref cannot reuse this + one's SBOM and re-runs cdxgen, not that this scan is marked failed for a + bookkeeping write. + """ + if fingerprint is None: + return + try: + with sync_session_scope() as session: + scan = session.get(Scan, scan_uuid) + if scan is not None: + scan.dependency_fingerprint = fingerprint + session.commit() + except Exception: # noqa: BLE001 (persistence is best-effort, never fatal) + log.warning( + "scan_dependency_fingerprint_persist_failed", + scan_id=str(scan_uuid), + exc_info=True, + ) + + +def _find_reusable_prior_scan( + *, + project_id: uuid.UUID, + ref: str | None, + scan_uuid: uuid.UUID, + fingerprint: str | None, +) -> uuid.UUID | None: + """S8: the prior succeeded scan this scan's SBOM generation can reuse. + + Returns the prior scan's id when the LATEST succeeded scan for this exact + ``(project_id, ref)`` carries a non-NULL ``dependency_fingerprint`` equal + to *fingerprint*, or ``None`` otherwise, including when *fingerprint* + itself is ``None`` (an un-fingerprintable scan is never a reuse + candidate: comparing ``None`` to ``None`` would call two DIFFERENT + un-fingerprinted trees "unchanged", which is exactly the false-positive + ``models.scan_fingerprint`` was written to avoid). + + "Latest succeeded scan for this (project, ref)" is the same row + ``tasks._scan_pipeline.mark_succeeded``'s ``supersede_prior_ref_scans`` + call already treats as authoritative for the ref, the scan a viewer + currently sees as "the" result. Comparing against anything older would + let a fingerprint match against a snapshot the retention model itself no + longer considers current. + """ + if fingerprint is None: + return None + with sync_session_scope() as session: + ref_filter = Scan.ref.is_(None) if ref is None else Scan.ref == ref + row = session.execute( + select(Scan.id, Scan.dependency_fingerprint) + .where( + Scan.project_id == project_id, + ref_filter, + Scan.status == "succeeded", + Scan.id != scan_uuid, + ) + .order_by(Scan.created_at.desc()) + .limit(1) + ).first() + if row is None: + return None + prior_scan_id: uuid.UUID = row[0] + prior_fingerprint: str | None = row[1] + if prior_fingerprint is None or prior_fingerprint != fingerprint: + return None + return prior_scan_id + + +def _reuse_prior_sbom( + *, + scan_uuid: uuid.UUID, + project_id: uuid.UUID, + prior_scan_id: uuid.UUID, + workspace: Path, +) -> cdxgen_adapter.CdxgenResult | None: + """S8: extract *prior_scan_id*'s preserved SBOM in place of running cdxgen. + + Writes into THIS scan's ``workspace / "cdxgen"`` directory (never the + prior scan's own workspace: that scan's workspace was already reclaimed + in its own ``finally`` block by the time this one runs), so the returned + ``CdxgenResult.sbom_path`` is exactly the shape the rest of the pipeline + already expects: a file inside the CURRENT scan's own workspace, cleaned + up by the current scan's own ``finally: shutil.rmtree(workspace)``. + + The extracted bytes are the FINAL SBOM the prior scan produced, + cocoapods-merged, scope-filtered, and metadata-stamped already (that is + what ``_preserve_source_tree`` archives; ``extract_preserved_sbom`` reads + the same tarball member the weekly rematch beat reads). Returns ``None`` + (never raises) on any extraction failure: a missing/corrupt preserved + tarball must never abort a scan; it only costs the fallback to the full + cdxgen path, exactly as if no fingerprint match had been found. + """ + out_dir = workspace / "cdxgen" + out_dir.mkdir(parents=True, exist_ok=True) + try: + sbom_path = extract_preserved_sbom( + scan_id=prior_scan_id, project_id=project_id, dest_dir=out_dir + ) + sbom = json.loads(sbom_path.read_text(encoding="utf-8")) + except ( + FileNotFoundError, + PreservedSbomMissing, + PreservationTooLarge, + SourcePreservationError, + json.JSONDecodeError, + OSError, + ) as exc: + log.warning( + "scan_dependency_fingerprint_reuse_failed", + scan_id=str(scan_uuid), + prior_scan_id=str(prior_scan_id), + reason=type(exc).__name__, + error=str(exc)[:300], + ) + return None + if not isinstance(sbom, dict): + log.warning( + "scan_dependency_fingerprint_reuse_failed", + scan_id=str(scan_uuid), + prior_scan_id=str(prior_scan_id), + reason="sbom_not_a_json_object", + ) + return None + return cdxgen_adapter.CdxgenResult(sbom_path=sbom_path, sbom=sbom) def _merge_cocoapods_components( diff --git a/apps/backend/tests/integration/scan/test_scan_source_dependency_fingerprint_reuse.py b/apps/backend/tests/integration/scan/test_scan_source_dependency_fingerprint_reuse.py new file mode 100644 index 00000000..bfe791ac --- /dev/null +++ b/apps/backend/tests/integration/scan/test_scan_source_dependency_fingerprint_reuse.py @@ -0,0 +1,449 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 TRUSCA contributors +""" +S8 (concurrency-scaling-plan-2026-08-22.md §3.2): dependency-set-fingerprint +scan reuse, wired end to end. + +``models.scan_fingerprint.compute_scan_fingerprint`` (schema + pure function, +db-designer) is unit-tested in isolation +(``tests/unit/test_scan_fingerprint.py``). What is still missing, and what +this file pins, is the WIRING inside ``tasks.scan_source``: does a scan +actually consult the prior succeeded scan's fingerprint, actually skip +cdxgen when it matches, actually re-run vulnerability matching regardless, +and actually fall back to the full pipeline when the scanner version (or the +manifest set) changed? + +Drives ``tasks.scan_source.scan_source_task`` directly (NOT through Celery's +broker) against a real Postgres, exactly like +``tests/integration/scan/test_scan_source_pipeline_mock.py``. cdxgen runs in +``TRUSTEDOSS_SCAN_BACKEND=mock`` mode (a deterministic single-component SBOM, +see ``integrations.cdxgen._write_mock_sbom``); a counting wrapper around +``cdxgen_adapter.run_cdxgen`` is the oracle for "did cdxgen actually run". +``scan_inputs.collect_manifest_inventory`` is monkeypatched to a fixed +inventory so the fingerprint comparison is deterministic across scans without +needing a real git tree with real lockfiles. + +Regression contract asserted here (plan §4 S8 row, hardening rule 5, +lifecycle sequence, not single-action): + + 1. Same lockfile set, same scanner version, same scan-time config, twice on + the same (project, ref) → the SECOND scan takes the reuse path (cdxgen + is NOT re-invoked) and still runs Trivy matching. + 2. A scanner-version bump between the two scans → the fingerprint differs → + the second scan re-runs the full pipeline (cdxgen IS re-invoked). + 3. The reused scan's own SBOM artifact lives inside ITS OWN workspace + (never a path under the prior scan's already-reclaimed workspace), and + the reused scan's own workspace is cleaned up in `finally` exactly like + a full-pipeline scan's. + 4. The reuse path and the full path leave indistinguishable scan-row shape + (``status='succeeded'``, ``progress_percent=100``, + ``current_step='finalize'``, ``completed_at`` set): a client cannot + tell which path a succeeded scan took from the row alone. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import subprocess +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import pytest +from sqlalchemy import create_engine, select +from sqlalchemy.orm import Session, sessionmaker + +from models import Scan, ScanArtifact, ScanComponent +from tests._helpers import ( + make_membership, + make_organization, + make_project, + make_team, + make_user, +) + +BACKEND_ROOT = Path(__file__).resolve().parent.parent.parent.parent + +pytestmark = pytest.mark.integration + + +def _require_database_url() -> str: + url = os.getenv("DATABASE_URL") + if not url: + pytest.skip("DATABASE_URL not set, skip S8 reuse integration") + return url + + +@pytest.fixture(scope="module", autouse=True) +def _migrate_once() -> None: + _require_database_url() + result = subprocess.run( + ["alembic", "upgrade", "head"], + cwd=BACKEND_ROOT, + capture_output=True, + text=True, + timeout=120, + ) + if result.returncode != 0: + pytest.skip( + f"alembic upgrade head failed; S8 reuse integration cannot run\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + + +@pytest.fixture +def sync_session() -> Iterator[Session]: + from core.config import database_url_sync + + engine = create_engine(database_url_sync(), pool_pre_ping=True, future=True) + factory = sessionmaker(bind=engine, expire_on_commit=False, future=True) + session = factory() + try: + yield session + finally: + session.close() + engine.dispose() + + +# --------------------------------------------------------------------------- +# Seeding helpers +# --------------------------------------------------------------------------- + + +def _seed_project() -> uuid.UUID: + """One project, reused by every scan in a test (S8 compares WITHIN a + project + ref, so every scan under test must share one).""" + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + + from core.config import database_url + + async def _build() -> uuid.UUID: + engine = create_async_engine(database_url(), pool_pre_ping=True, future=True) + factory = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession) + async with factory() as s: + org = await make_organization(s) + team = await make_team(s, organization=org) + user = await make_user(s) + await make_membership(s, user=user, team=team, role="developer") + # git_url=None: same no-source placeholder path + # test_scan_source_pipeline_mock.py uses. The mock cdxgen backend + # emits its SBOM regardless of what _fetch_source produced. + project = await make_project(s, team=team, git_url=None) + project_id = project.id + await engine.dispose() + return project_id + + return asyncio.run(_build()) + + +def _seed_queued_scan( + session: Session, *, project_id: uuid.UUID, ref: str | None +) -> uuid.UUID: + """A fresh queued Scan row for an EXISTING project (sync insert, the scan + task itself only needs the row to exist, not the project/team graph).""" + scan = Scan( + project_id=project_id, + kind="source", + status="queued", + progress_percent=0, + scan_metadata={}, + ref=ref, + ) + session.add(scan) + session.commit() + session.refresh(scan) + return scan.id + + +# --------------------------------------------------------------------------- +# Determinism helpers: fixed manifest inventory, counted cdxgen, stub Trivy +# --------------------------------------------------------------------------- + + +def _fixed_manifest_inventory() -> dict[str, Any]: + """The exact shape ``services.scan_inputs.collect_manifest_inventory`` + returns. Identical across every call in a test ⇒ the fingerprint's + manifest input never varies on its own (only ``CDXGEN_VERSION`` / + scan-config toggles are allowed to move the fingerprint in these tests. + """ + return { + "files": [ + {"path": "package-lock.json", "size": 512, "sha256": "b" * 64}, + {"path": "package.json", "size": 128, "sha256": "c" * 64}, + ], + "count": 2, + "truncated": False, + } + + +def _pin_manifest_inventory(monkeypatch: pytest.MonkeyPatch) -> None: + import services.scan_inputs as scan_inputs_module + + monkeypatch.setattr( + scan_inputs_module, + "collect_manifest_inventory", + lambda _project_root: _fixed_manifest_inventory(), + ) + + +def _count_cdxgen_calls(monkeypatch: pytest.MonkeyPatch) -> list[int]: + """Wrap ``cdxgen_adapter.run_cdxgen`` to record each real invocation + while still delegating to the mock backend, so a reused scan's call + count stays at the PRIOR scan's total (proof cdxgen never ran again).""" + from integrations import cdxgen as cdxgen_adapter + + calls: list[int] = [] + original = cdxgen_adapter.run_cdxgen + + def _wrapped(*args: Any, **kwargs: Any) -> Any: + calls.append(1) + return original(*args, **kwargs) + + monkeypatch.setattr("tasks.scan_source.cdxgen_adapter.run_cdxgen", _wrapped) + return calls + + +def _stub_trivy(monkeypatch: pytest.MonkeyPatch) -> list[int]: + """Same empty-report stub as test_scan_source_pipeline_mock.py's + ``_stub_trivy_empty``, plus a call counter. S8's accuracy contract is + that Trivy matching reruns on EVERY scan, reused SBOM or not.""" + from integrations.trivy import TrivyResult + + calls: list[int] = [] + + def _fake_run( + sbom_path: Path, + output_dir: Path, + *, + timeout_seconds: int = 0, # noqa: ARG001 + backend: str | None = None, # noqa: ARG001 + **_kwargs: object, + ) -> TrivyResult: + calls.append(1) + output_dir.mkdir(parents=True, exist_ok=True) + report_path = output_dir / "trivy-sbom.json" + report = { + "SchemaVersion": 2, + "ArtifactName": str(sbom_path), + "ArtifactType": "cyclonedx", + "Results": [], + } + report_path.write_text(json.dumps(report), encoding="utf-8") + return TrivyResult(report_path=report_path, report=report) + + monkeypatch.setattr("tasks.scan_source.run_trivy_sbom", _fake_run) + return calls + + +def _component_purls(session: Session, scan_id: uuid.UUID) -> set[str]: + from models import Component, ComponentVersion + + rows = session.execute( + select(Component.purl) + .join(ComponentVersion, ComponentVersion.component_id == Component.id) + .join(ScanComponent, ScanComponent.component_version_id == ComponentVersion.id) + .where(ScanComponent.scan_id == scan_id) + ).all() + return {r[0] for r in rows} + + +# --------------------------------------------------------------------------- +# 1. Same fingerprint twice → second scan reuses, cdxgen not re-invoked +# --------------------------------------------------------------------------- + + +def test_second_scan_with_unchanged_fingerprint_reuses_prior_sbom( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, sync_session: Session +) -> None: + monkeypatch.setenv("TRUSTEDOSS_SCAN_BACKEND", "mock") + monkeypatch.setenv("WORKSPACE_HOST_PATH", str(tmp_path)) + monkeypatch.setenv("CDXGEN_VERSION", "12.3.3") + _pin_manifest_inventory(monkeypatch) + cdxgen_calls = _count_cdxgen_calls(monkeypatch) + trivy_calls = _stub_trivy(monkeypatch) + + project_id = _seed_project() + + from tasks.scan_source import scan_source_task + + scan1_id = _seed_queued_scan(sync_session, project_id=project_id, ref="main") + result1 = scan_source_task.apply(args=[str(scan1_id)]) + assert result1.successful(), f"scan1 failed: {result1.traceback}" + + sync_session.expire_all() + scan1 = sync_session.execute(select(Scan).where(Scan.id == scan1_id)).scalar_one() + assert scan1.status == "succeeded" + assert scan1.dependency_fingerprint is not None + assert len(cdxgen_calls) == 1 + assert len(trivy_calls) == 1 + + scan2_id = _seed_queued_scan(sync_session, project_id=project_id, ref="main") + result2 = scan_source_task.apply(args=[str(scan2_id)]) + assert result2.successful(), f"scan2 failed: {result2.traceback}" + + sync_session.expire_all() + scan2 = sync_session.execute(select(Scan).where(Scan.id == scan2_id)).scalar_one() + assert scan2.status == "succeeded" + # cdxgen must NOT have run a second time: the reuse path took over. + assert len(cdxgen_calls) == 1, "cdxgen re-ran on scan2 despite an unchanged fingerprint" + # Trivy matching MUST still have run on scan2: reuse only skips cdxgen. + assert len(trivy_calls) == 2, "vulnerability matching must re-run on every scan" + + # Same fingerprint value (the reused scan's own bytes are identical). + assert scan2.dependency_fingerprint == scan1.dependency_fingerprint + + # The reused document produced the SAME component graph. + assert _component_purls(sync_session, scan1_id) == _component_purls( + sync_session, scan2_id + ) + assert _component_purls(sync_session, scan2_id) == {"pkg:npm/example"} + + # scan2's own SBOM artifact lives under scan2's OWN id, never scan1's, + # workspace isolation even on a reuse extraction. + scan2_sbom = sync_session.execute( + select(ScanArtifact).where( + ScanArtifact.scan_id == scan2_id, ScanArtifact.kind == "sbom_cyclonedx" + ) + ).scalar_one() + assert str(scan2_id) in scan2_sbom.storage_path + assert str(scan1_id) not in scan2_sbom.storage_path + + # finally: shutil.rmtree(workspace) reclaimed scan2's workspace exactly + # like a full-pipeline scan's. Reuse must not leak a workspace dir. + assert not (tmp_path / str(scan2_id)).exists() + + # Row-shape parity: a reused scan looks exactly like a full-pipeline scan + # from the outside. + assert scan2.progress_percent == 100 + assert scan2.current_step == "finalize" + assert scan2.completed_at is not None + assert scan2.error_message is None + + +# --------------------------------------------------------------------------- +# 2. Scanner version bump → fingerprint differs → full pipeline re-runs +# --------------------------------------------------------------------------- + + +def test_scanner_version_bump_forces_the_full_pipeline_to_rerun( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, sync_session: Session +) -> None: + monkeypatch.setenv("TRUSTEDOSS_SCAN_BACKEND", "mock") + monkeypatch.setenv("WORKSPACE_HOST_PATH", str(tmp_path)) + _pin_manifest_inventory(monkeypatch) + cdxgen_calls = _count_cdxgen_calls(monkeypatch) + _stub_trivy(monkeypatch) + + project_id = _seed_project() + + from tasks.scan_source import scan_source_task + + monkeypatch.setenv("CDXGEN_VERSION", "12.3.3") + scan1_id = _seed_queued_scan(sync_session, project_id=project_id, ref="main") + result1 = scan_source_task.apply(args=[str(scan1_id)]) + assert result1.successful() + assert len(cdxgen_calls) == 1 + + sync_session.expire_all() + scan1 = sync_session.execute(select(Scan).where(Scan.id == scan1_id)).scalar_one() + assert scan1.dependency_fingerprint is not None + + # Simulate a worker image upgrade between the two scans: same manifest + # inventory, different pinned cdxgen version. + monkeypatch.setenv("CDXGEN_VERSION", "13.0.0") + scan2_id = _seed_queued_scan(sync_session, project_id=project_id, ref="main") + result2 = scan_source_task.apply(args=[str(scan2_id)]) + assert result2.successful() + + sync_session.expire_all() + scan2 = sync_session.execute(select(Scan).where(Scan.id == scan2_id)).scalar_one() + assert scan2.status == "succeeded" + # cdxgen DID run again: the version bump must never read as "unchanged". + assert len(cdxgen_calls) == 2, ( + "a scanner-version bump must force cdxgen to re-run, not reuse the " + "prior version's SBOM" + ) + assert scan2.dependency_fingerprint is not None + assert scan2.dependency_fingerprint != scan1.dependency_fingerprint + + +# --------------------------------------------------------------------------- +# 3. Different ref → no reuse (S8 compares within (project, ref) only) +# --------------------------------------------------------------------------- + + +def test_different_ref_on_the_same_project_does_not_reuse( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, sync_session: Session +) -> None: + monkeypatch.setenv("TRUSTEDOSS_SCAN_BACKEND", "mock") + monkeypatch.setenv("WORKSPACE_HOST_PATH", str(tmp_path)) + monkeypatch.setenv("CDXGEN_VERSION", "12.3.3") + _pin_manifest_inventory(monkeypatch) + cdxgen_calls = _count_cdxgen_calls(monkeypatch) + _stub_trivy(monkeypatch) + + project_id = _seed_project() + + from tasks.scan_source import scan_source_task + + scan1_id = _seed_queued_scan(sync_session, project_id=project_id, ref="main") + scan_source_task.apply(args=[str(scan1_id)]) + assert len(cdxgen_calls) == 1 + + # A PR branch scan on the SAME project, DIFFERENT ref: no prior succeeded + # scan exists for (project_id, "pr-7") yet, so this must run the full + # pipeline even though the manifest/scanner/config are all identical. + scan2_id = _seed_queued_scan(sync_session, project_id=project_id, ref="pr-7") + result2 = scan_source_task.apply(args=[str(scan2_id)]) + assert result2.successful() + assert len(cdxgen_calls) == 2, "a different ref must never reuse another ref's SBOM" + + +# --------------------------------------------------------------------------- +# 4. Idempotent retry: re-running an already-succeeded reused scan is a no-op +# --------------------------------------------------------------------------- + + +def test_rerunning_a_succeeded_reused_scan_is_still_idempotent( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, sync_session: Session +) -> None: + monkeypatch.setenv("TRUSTEDOSS_SCAN_BACKEND", "mock") + monkeypatch.setenv("WORKSPACE_HOST_PATH", str(tmp_path)) + monkeypatch.setenv("CDXGEN_VERSION", "12.3.3") + _pin_manifest_inventory(monkeypatch) + cdxgen_calls = _count_cdxgen_calls(monkeypatch) + _stub_trivy(monkeypatch) + + project_id = _seed_project() + + from tasks.scan_source import scan_source_task + + scan1_id = _seed_queued_scan(sync_session, project_id=project_id, ref="main") + scan_source_task.apply(args=[str(scan1_id)]) + + scan2_id = _seed_queued_scan(sync_session, project_id=project_id, ref="main") + scan_source_task.apply(args=[str(scan2_id)]) + sync_session.expire_all() + scan2_first = sync_session.execute( + select(Scan).where(Scan.id == scan2_id) + ).scalar_one() + assert scan2_first.status == "succeeded" + completed_at_first = scan2_first.completed_at + components_first = _component_purls(sync_session, scan2_id) + + # Celery acks_late + worker-restart re-entry on the SAME (already- + # succeeded) reused scan: task-level idempotency short-circuits before + # the reuse decision even runs again. + scan_source_task.apply(args=[str(scan2_id)]) + sync_session.expire_all() + scan2_again = sync_session.execute( + select(Scan).where(Scan.id == scan2_id) + ).scalar_one() + assert scan2_again.status == "succeeded" + assert scan2_again.completed_at == completed_at_first + assert _component_purls(sync_session, scan2_id) == components_first + # No extra cdxgen invocation, and no extra components accumulated. + assert len(cdxgen_calls) == 1 diff --git a/apps/backend/tests/integration/test_scan_dependency_fingerprint_migration.py b/apps/backend/tests/integration/test_scan_dependency_fingerprint_migration.py new file mode 100644 index 00000000..7c7ca1f1 --- /dev/null +++ b/apps/backend/tests/integration/test_scan_dependency_fingerprint_migration.py @@ -0,0 +1,220 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 TRUSCA contributors +""" +Integration tests for migration 0070 (``scans.dependency_fingerprint``). + +S8 (concurrency-scaling-plan-2026-08-22.md §3.2, unit 29) is a "연속" +(multi-PR) unit; this session ships the schema step only. What has to be +true of THAT step, independent of the reuse-decision logic a later revision +adds: + + - the column exists after ``alembic upgrade head`` with the shape the + model declares (nullable, fixed-length string sized for a SHA-256 hex + digest), and a real digest round-trips through it unchanged; + - the migration's own ``downgrade()`` fails loudly rather than silently + dropping the column (CLAUDE.md §6 forward-only policy), both at the + Python level (calling the function directly) and through the real + ``alembic downgrade`` CLI a rollback attempt would actually invoke. +""" + +from __future__ import annotations + +import importlib.util +import os +import subprocess +import uuid +from collections.abc import Iterator +from pathlib import Path + +import pytest +from sqlalchemy import create_engine, text +from sqlalchemy.orm import Session, sessionmaker + +from models import Organization, Project, Scan, Team +from models.scan_fingerprint import compute_scan_fingerprint + +BACKEND_ROOT = Path(__file__).resolve().parent.parent.parent +_MIGRATION_0070_PATH = ( + BACKEND_ROOT / "alembic" / "versions" / "0070_scan_dependency_fingerprint.py" +) + +pytestmark = pytest.mark.integration + + +def _sync_url() -> str: + url = os.getenv("DATABASE_URL") + if not url: + pytest.skip("DATABASE_URL not set, skip alembic integration test") + return url.replace("postgresql+asyncpg://", "postgresql+psycopg2://") + + +@pytest.fixture(scope="module", autouse=True) +def _migrate_to_head() -> None: + if not os.getenv("DATABASE_URL"): + pytest.skip("DATABASE_URL not set") + result = subprocess.run( + ["alembic", "upgrade", "head"], + cwd=BACKEND_ROOT, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + pytest.skip(f"alembic upgrade head failed: {result.stderr[-400:]}") + + +@pytest.fixture +def session() -> Iterator[Session]: + engine = create_engine(_sync_url(), future=True) + factory = sessionmaker(bind=engine, future=True) + with factory() as s: + yield s + engine.dispose() + + +def _seed_project(session: Session) -> uuid.UUID: + suffix = uuid.uuid4().hex[:8] + org = Organization(name=f"S8 Org {suffix}", slug=f"s8-org-{suffix}") + session.add(org) + session.flush() + team = Team(organization_id=org.id, name=f"S8 Team {suffix}", slug=f"s8-team-{suffix}") + session.add(team) + session.flush() + project = Project(team_id=team.id, name=f"S8 Proj {suffix}", slug=f"s8-proj-{suffix}") + session.add(project) + session.commit() + return project.id + + +def _load_migration_0070(): + """Load the migration file directly by path. + + ``alembic.versions`` cannot be imported as a dotted module name: the + installed ``alembic`` tooling package already owns the top-level + ``alembic`` name on ``sys.path``, and it has no ``versions`` submodule of + its own, so ``importlib.import_module("alembic.versions.0070_...")`` + resolves against the wrong package and raises ``ModuleNotFoundError``. + Loading by file path sidesteps the name collision entirely. + """ + spec = importlib.util.spec_from_file_location( + "trusca_migration_0070", _MIGRATION_0070_PATH + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_dependency_fingerprint_column_shape(session: Session) -> None: + row = session.execute( + text( + "SELECT data_type, character_maximum_length, is_nullable " + "FROM information_schema.columns " + "WHERE table_name = 'scans' AND column_name = 'dependency_fingerprint'" + ) + ).one() + data_type, max_length, is_nullable = row + + assert data_type == "character varying" + assert max_length == 64 # exactly a SHA-256 hex digest's length + assert is_nullable == "YES" # existing scans predate this column + + +def test_dependency_fingerprint_round_trips_a_computed_digest(session: Session) -> None: + """A digest ``compute_scan_fingerprint`` actually returns writes and + reads back unchanged: no silent truncation or type coercion.""" + digest = compute_scan_fingerprint( + manifest_inventory={ + "files": [{"path": "go.sum", "size": 10, "sha256": "a" * 64}], + "count": 1, + "truncated": False, + }, + scanner_version="12.3.3", + scan_config={"cdxgen_spec_version": "1.5"}, + ) + assert digest is not None + assert len(digest) == 64 + + project_id = _seed_project(session) + scan = Scan( + project_id=project_id, + kind="source", + status="succeeded", + dependency_fingerprint=digest, + ) + session.add(scan) + session.commit() + session.refresh(scan) + + assert scan.dependency_fingerprint == digest + + reloaded = session.get(Scan, scan.id) + assert reloaded is not None + assert reloaded.dependency_fingerprint == digest + + +def test_dependency_fingerprint_defaults_to_null(session: Session) -> None: + """A scan created without a fingerprint (every pre-0070 scan, and any + scan the pipeline could not fingerprint) stores NULL, not an empty + string or a sentinel: NULL is what the model docstring's "never equal + to another NULL" contract depends on. + """ + project_id = _seed_project(session) + scan = Scan(project_id=project_id, kind="source", status="succeeded") + session.add(scan) + session.commit() + session.refresh(scan) + + assert scan.dependency_fingerprint is None + + +def test_migration_0070_downgrade_raises_not_implemented() -> None: + """Calling the migration module's ``downgrade()`` directly: no DB or + Alembic runtime needed, since the function raises before touching ``op``. + """ + module = _load_migration_0070() + + with pytest.raises(NotImplementedError): + module.downgrade() + + +def test_alembic_downgrade_cli_fails_on_0070() -> None: + """The forward-only contract as an operator would actually hit it: an + ``alembic downgrade`` attempt from head must exit non-zero, not silently + drop the column. + """ + current = subprocess.run( + ["alembic", "current"], + cwd=BACKEND_ROOT, + capture_output=True, + text=True, + timeout=30, + ) + assert current.returncode == 0, current.stderr + if "0070" not in current.stdout: + pytest.skip( + "head is past 0070 already, a later migration owns the downgrade " + "boundary the CLI would hit first; 0070's own raise is covered by " + "the direct-call test above" + ) + + downgrade = subprocess.run( + ["alembic", "downgrade", "-1"], + cwd=BACKEND_ROOT, + capture_output=True, + text=True, + timeout=60, + ) + assert downgrade.returncode != 0 + assert "NotImplementedError" in downgrade.stderr or "forward-only" in downgrade.stderr + + # Restore head so this test does not leave the shared DB mid-migration + # for whatever runs next. + restore = subprocess.run( + ["alembic", "upgrade", "head"], + cwd=BACKEND_ROOT, + capture_output=True, + text=True, + timeout=60, + ) + assert restore.returncode == 0, restore.stderr diff --git a/apps/backend/tests/unit/tasks/test_scan_chaos.py b/apps/backend/tests/unit/tasks/test_scan_chaos.py index fde1116e..250c1a90 100644 --- a/apps/backend/tests/unit/tasks/test_scan_chaos.py +++ b/apps/backend/tests/unit/tasks/test_scan_chaos.py @@ -45,6 +45,8 @@ class _FakeScan: project_id = uuid.uuid4() scan_metadata = None id = scan_uuid + # S8: scan_source_task snapshots scan.ref before the pipeline runs. + ref = None class _FakeProject: id = _FakeScan.project_id diff --git a/apps/backend/tests/unit/tasks/test_scan_source_dependency_fingerprint_fallback.py b/apps/backend/tests/unit/tasks/test_scan_source_dependency_fingerprint_fallback.py new file mode 100644 index 00000000..a98c7457 --- /dev/null +++ b/apps/backend/tests/unit/tasks/test_scan_source_dependency_fingerprint_fallback.py @@ -0,0 +1,303 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 TRUSCA contributors +""" +S8 (concurrency-scaling-plan-2026-08-22.md §3.2): reuse-extraction failure +fallback, unit-tested in isolation from the full pipeline. + +``tests/integration/scan/test_scan_source_dependency_fingerprint_reuse.py`` +drives the SUCCESSFUL reuse path end to end against a real Postgres. What is +still missing, and what this file pins, is the defensive branch: a +fingerprint match whose preserved tarball is gone, corrupt, or carries a +non-object JSON payload by the time the reuse extraction actually runs (the +tarball was written at a DIFFERENT time than the fingerprint comparison. The +retention beat, an admin purge, or plain disk trouble can remove it in +between). ``_reuse_prior_sbom`` must return ``None`` (never raise) on every +one of those, and ``_run_pipeline``'s caller must fall back to the full +cdxgen path exactly as if no fingerprint match had been found at all. S8 is +an optimization, never a correctness dependency. +""" + +from __future__ import annotations + +import json +import uuid +from pathlib import Path + +import pytest + +import tasks.scan_source as mod +from services.source_preservation_service import ( + PreservationTooLarge, + PreservedSbomMissing, + SourcePreservationError, +) + +# --------------------------------------------------------------------------- +# _reuse_prior_sbom: success +# --------------------------------------------------------------------------- + + +def test_reuse_prior_sbom_returns_the_extracted_document( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + scan_uuid = uuid.uuid4() + expected_project_id = uuid.uuid4() + expected_prior_scan_id = uuid.uuid4() + workspace = tmp_path / "workspace" + + sbom = {"bomFormat": "CycloneDX", "components": [{"purl": "pkg:npm/x@1.0.0"}]} + + def _fake_extract(*, scan_id: uuid.UUID, project_id: uuid.UUID, dest_dir: Path) -> Path: + assert scan_id == expected_prior_scan_id + assert project_id == expected_project_id + dest_dir.mkdir(parents=True, exist_ok=True) + out = dest_dir / "cdxgen.cdx.json" + out.write_text(json.dumps(sbom), encoding="utf-8") + return out + + monkeypatch.setattr(mod, "extract_preserved_sbom", _fake_extract) + + result = mod._reuse_prior_sbom( + scan_uuid=scan_uuid, + project_id=expected_project_id, + prior_scan_id=expected_prior_scan_id, + workspace=workspace, + ) + + assert result is not None + assert result.sbom == sbom + # Extracted into THIS scan's own workspace, not some shared/global path. + assert str(workspace) in str(result.sbom_path) + + +# --------------------------------------------------------------------------- +# _reuse_prior_sbom: every failure mode returns None, never raises +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "raised", + [ + FileNotFoundError("preserved tarball missing"), + PreservedSbomMissing("no sbom member"), + PreservationTooLarge("sbom exceeds cap"), + SourcePreservationError("tar corrupt"), + OSError("disk read failed"), + ], +) +def test_reuse_prior_sbom_extraction_failure_returns_none( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, raised: Exception +) -> None: + def _boom(*, scan_id: uuid.UUID, project_id: uuid.UUID, dest_dir: Path) -> Path: + raise raised + + monkeypatch.setattr(mod, "extract_preserved_sbom", _boom) + + result = mod._reuse_prior_sbom( + scan_uuid=uuid.uuid4(), + project_id=uuid.uuid4(), + prior_scan_id=uuid.uuid4(), + workspace=tmp_path / "workspace", + ) + + assert result is None + + +def test_reuse_prior_sbom_corrupt_json_returns_none( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The tarball extracted fine, but the bytes inside are not valid JSON, + a truncated write, or a hostile/corrupted archive.""" + + def _fake_extract(*, scan_id: uuid.UUID, project_id: uuid.UUID, dest_dir: Path) -> Path: + dest_dir.mkdir(parents=True, exist_ok=True) + out = dest_dir / "cdxgen.cdx.json" + out.write_text("{not valid json", encoding="utf-8") + return out + + monkeypatch.setattr(mod, "extract_preserved_sbom", _fake_extract) + + result = mod._reuse_prior_sbom( + scan_uuid=uuid.uuid4(), + project_id=uuid.uuid4(), + prior_scan_id=uuid.uuid4(), + workspace=tmp_path / "workspace", + ) + + assert result is None + + +def test_reuse_prior_sbom_non_object_json_returns_none( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Valid JSON, but not a CycloneDX document (a bare JSON array). Must + not be accepted as an ``sbom: dict`` just because ``json.loads`` succeeded.""" + + def _fake_extract(*, scan_id: uuid.UUID, project_id: uuid.UUID, dest_dir: Path) -> Path: + dest_dir.mkdir(parents=True, exist_ok=True) + out = dest_dir / "cdxgen.cdx.json" + out.write_text(json.dumps([1, 2, 3]), encoding="utf-8") + return out + + monkeypatch.setattr(mod, "extract_preserved_sbom", _fake_extract) + + result = mod._reuse_prior_sbom( + scan_uuid=uuid.uuid4(), + project_id=uuid.uuid4(), + prior_scan_id=uuid.uuid4(), + workspace=tmp_path / "workspace", + ) + + assert result is None + + +# --------------------------------------------------------------------------- +# _record_input_manifests: a persist failure returns None (not the collected +# inventory), so S8's fingerprint computation never treats a scan whose +# manifest inventory it could not actually store as a fingerprintable one. +# --------------------------------------------------------------------------- + + +def test_record_input_manifests_returns_none_when_persistence_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from contextlib import contextmanager + + inventory = { + "files": [{"path": "package.json", "size": 1, "sha256": "a" * 64}], + "count": 1, + "truncated": False, + } + monkeypatch.setattr( + "tasks.scan_source.scan_inputs.collect_manifest_inventory", + lambda _root: inventory, + ) + + @contextmanager + def _boom_scope(): # type: ignore[no-untyped-def] + raise RuntimeError("db unavailable") + yield # pragma: no cover (unreachable, satisfies generator shape) + + monkeypatch.setattr(mod, "sync_session_scope", _boom_scope) + + result = mod._record_input_manifests(uuid.uuid4(), Path("/nonexistent")) + + assert result is None + + +# --------------------------------------------------------------------------- +# _persist_dependency_fingerprint: best-effort, never raises +# --------------------------------------------------------------------------- + + +def test_persist_dependency_fingerprint_noop_when_none() -> None: + """A None fingerprint (un-fingerprintable scan) must not even try a + write, asserted by NOT monkeypatching sync_session_scope at all; a call + into it would raise (no DB in this test).""" + mod._persist_dependency_fingerprint(uuid.uuid4(), None) + + +def test_persist_dependency_fingerprint_swallows_a_db_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from contextlib import contextmanager + + @contextmanager + def _boom_scope(): # type: ignore[no-untyped-def] + raise RuntimeError("db unavailable") + yield # pragma: no cover (unreachable, satisfies generator shape) + + monkeypatch.setattr(mod, "sync_session_scope", _boom_scope) + + # Must not raise: a bookkeeping write failing here must never fail an + # otherwise-successful scan. + mod._persist_dependency_fingerprint(uuid.uuid4(), "a" * 64) + + +# --------------------------------------------------------------------------- +# _run_pipeline: a failed reuse extraction falls back to the full path +# --------------------------------------------------------------------------- + + +def test_run_pipeline_falls_back_to_full_cdxgen_when_reuse_extraction_fails( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """``_find_reusable_prior_scan`` found a fingerprint match, but + ``_reuse_prior_sbom`` could not actually extract it (tarball reclaimed + between the fingerprint write and this scan). The pipeline must still + reach the full cdxgen call, not abort or silently skip SBOM generation.""" + scan_uuid = uuid.uuid4() + project_id = uuid.uuid4() + workspace = tmp_path / str(scan_uuid) + + monkeypatch.setattr(mod, "_set_stage", lambda *a, **k: None) + monkeypatch.setattr(mod, "_fetch_source", lambda **k: workspace / "source") + monkeypatch.setattr(mod, "_resolve_project_root", lambda source_dir: source_dir) + monkeypatch.setattr(mod, "_detect_and_record_env", lambda *a, **k: "unknown") + monkeypatch.setattr(mod, "_record_input_manifests", lambda *a, **k: { + "files": [{"path": "package.json", "size": 1, "sha256": "a" * 64}], + "count": 1, + "truncated": False, + }) + monkeypatch.setattr(mod, "cdxgen_scanner_version", lambda: "12.3.3") + monkeypatch.setattr(mod, "cdxgen_spec_version", lambda: "1.5") + monkeypatch.setattr(mod, "cdxgen_fetch_license", lambda: False) + monkeypatch.setattr(mod, "scan_scope_filter_enabled", lambda: True) + monkeypatch.setattr(mod, "scan_scope_filter_maven_enabled", lambda: True) + monkeypatch.setattr(mod, "scan_scope_filter_node_enabled", lambda: True) + + # A reuse candidate exists... + monkeypatch.setattr(mod, "_find_reusable_prior_scan", lambda **k: uuid.uuid4()) + # ...but extraction fails (the defensive branch under test). + monkeypatch.setattr(mod, "_reuse_prior_sbom", lambda **k: None) + + full_pipeline_calls: list[str] = [] + + class _FakeExecutor: + def generate_sbom(self, request, *, prep, stage, line_callback): # type: ignore[no-untyped-def] + full_pipeline_calls.append("generate_sbom") + out_dir = request.output_dir + out_dir.mkdir(parents=True, exist_ok=True) + sbom_path = out_dir / "cdxgen.cdx.json" + sbom = {"bomFormat": "CycloneDX", "components": []} + sbom_path.write_text(json.dumps(sbom), encoding="utf-8") + + class _Result: + pass + + r = _Result() + r.sbom_path = sbom_path # type: ignore[attr-defined] + r.sbom = sbom # type: ignore[attr-defined] + return r + + monkeypatch.setattr( + "tasks.scan_source.scan_executor.get_executor", lambda: _FakeExecutor() + ) + monkeypatch.setattr(mod, "_merge_cocoapods_components", lambda **k: None) + monkeypatch.setattr(mod, "_apply_scope_filter", lambda **k: None) + monkeypatch.setattr(mod, "_stamp_document_metadata", lambda **k: None) + monkeypatch.setattr(mod, "_persist_artifact", lambda *a, **k: None) + monkeypatch.setattr(mod, "_sign_sbom", lambda **k: False) + + def _stop_here(**_k: object) -> None: + raise _StopAtScancode() + + class _StopAtScancode(Exception): + pass + + monkeypatch.setattr("tasks.scan_source.scancode_adapter.run_scancode", _stop_here) + + with pytest.raises(_StopAtScancode): + mod._run_pipeline( + scan_uuid=scan_uuid, + project_id=project_id, + workspace=workspace, + git_url=None, + scan_metadata={}, + ref="main", + ) + + assert full_pipeline_calls == ["generate_sbom"], ( + "a failed reuse extraction must fall back to the real cdxgen call, " + "not silently skip SBOM generation" + ) diff --git a/apps/backend/tests/unit/tasks/test_scan_source_load_test_delay.py b/apps/backend/tests/unit/tasks/test_scan_source_load_test_delay.py index 4f45e673..7e04499a 100644 --- a/apps/backend/tests/unit/tasks/test_scan_source_load_test_delay.py +++ b/apps/backend/tests/unit/tasks/test_scan_source_load_test_delay.py @@ -40,6 +40,8 @@ class _FakeScan: project_id = uuid.uuid4() scan_metadata: dict[str, Any] | None = None id: uuid.UUID + # S8: scan_source_task snapshots scan.ref before the pipeline runs. + ref: str | None = None class _FakeProject: diff --git a/apps/backend/tests/unit/tasks/test_scan_source_scope_filter.py b/apps/backend/tests/unit/tasks/test_scan_source_scope_filter.py index 7936dbd0..f102b77a 100644 --- a/apps/backend/tests/unit/tasks/test_scan_source_scope_filter.py +++ b/apps/backend/tests/unit/tasks/test_scan_source_scope_filter.py @@ -308,23 +308,44 @@ def test_apply_scope_filter_rewrite_failure_keeps_memory_and_disk_unfiltered( def _call_order(func_name: str, module_path: Path) -> list[str]: - """First-call order of named functions inside ``func_name`` (AST walk).""" + """First-call order of named functions inside ``func_name`` (AST walk). + + Pre-order depth-first over ``ast.iter_child_nodes`` (source/field order), + NOT ``ast.walk`` (breadth-first): ``ast.walk`` visits every statement at + the SAME nesting depth before descending into any of their bodies, so a + call inside an ``if``/``else`` branch is reported as happening AFTER a + call that comes later in the source but sits one level shallower (e.g. a + top-level statement right after the ``if``/``else``). S8 + (concurrency-scaling-plan-2026-08-22.md §3.2) branches + ``_merge_cocoapods_components`` / ``_apply_scope_filter`` / + ``_stamp_document_metadata`` into the non-reuse ``else`` arm, one level + deeper than the ``_persist_artifact`` call that follows the whole + if/else: an ``ast.walk``-based order would misreport that as + "``_persist_artifact`` first" even though it always runs after, on every + branch. DFS in source order does not have that failure mode. + """ tree = ast.parse(module_path.read_text(encoding="utf-8")) + + def _name_of(call: ast.Call) -> str | None: + callee = call.func + if isinstance(callee, ast.Name): + return callee.id + if isinstance(callee, ast.Attribute): + return callee.attr + return None + + def _visit(node: ast.AST, order: list[str]) -> None: + if isinstance(node, ast.Call): + name = _name_of(node) + if name and name not in order: + order.append(name) + for child in ast.iter_child_nodes(node): + _visit(child, order) + for node in ast.walk(tree): if isinstance(node, ast.FunctionDef) and node.name == func_name: order: list[str] = [] - for call in ast.walk(node): - if isinstance(call, ast.Call): - callee = call.func - name = ( - callee.id - if isinstance(callee, ast.Name) - else callee.attr - if isinstance(callee, ast.Attribute) - else None - ) - if name and name not in order: - order.append(name) + _visit(node, order) return order raise AssertionError(f"{func_name} not found in {module_path}") diff --git a/apps/backend/tests/unit/tasks/test_scan_timeout.py b/apps/backend/tests/unit/tasks/test_scan_timeout.py index 89110be4..a359ab12 100644 --- a/apps/backend/tests/unit/tasks/test_scan_timeout.py +++ b/apps/backend/tests/unit/tasks/test_scan_timeout.py @@ -212,6 +212,9 @@ class _FakeScan: # pipeline; the source path carries no image_ref, so None → {}. scan_metadata = None id = scan_uuid + # S8: scan_source_task also snapshots scan.ref before the pipeline + # runs (unrelated to this timeout scenario, but read unconditionally). + ref = None class _FakeProject: id = _FakeScan.project_id diff --git a/apps/backend/tests/unit/test_scan_fingerprint.py b/apps/backend/tests/unit/test_scan_fingerprint.py new file mode 100644 index 00000000..80f85cd2 --- /dev/null +++ b/apps/backend/tests/unit/test_scan_fingerprint.py @@ -0,0 +1,348 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 TRUSCA contributors +""" +Unit tests for :mod:`models.scan_fingerprint`. + +S8 (concurrency-scaling-plan-2026-08-22.md §3.2) reuses a scan's preserved +SBOM instead of re-running cdxgen when the fingerprint is unchanged. The +properties that matter, in order of how badly a violation would hurt: + + 1. Determinism: the same inputs, computed twice, must be the digest twice. + Reuse compares two stored digests; a function that is not deterministic + makes every scan look "changed" (safe but pointless, see the load- + bearing test below for why this is more than academic) or, if + insertion-order-dependent, could make two DIFFERENT trees collide. + 2. A scanner-version bump changes the digest, the plan's explicit accuracy + requirement (§3.2: "지문은 잠금 파일만이 아니라 스캐너 버전과 스캔 설정까지 + 포함해야 한다. 그러지 않으면 스캐너를 올린 뒤에도 옛 SBOM을 재사용한다"). + A miss here means a future reuse-decision revision silently serves a + stale SBOM after every worker upgrade. + 3. A scan-config change (spec version, scope-filter toggles) changes the + digest, for the same reason. + 4. A lockfile content change changes the digest: the baseline the whole + feature exists to detect. + 5. The function refuses to answer (returns None) when the inventory cannot + be trusted to describe the whole tree: no inventory, a truncated walk, + or an unhashed file. A None that a caller mistook for "matches another + None" would silently reuse an SBOM for a tree nobody actually compared. +""" + +from __future__ import annotations + +import copy + +from models.scan_fingerprint import ( + FINGERPRINT_SCHEMA_VERSION, + compute_scan_fingerprint, +) + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +def _inventory(*files: tuple[str, str]) -> dict[str, object]: + """Build an inventory in the exact shape ``collect_manifest_inventory`` returns.""" + entries = [{"path": path, "size": 123, "sha256": digest} for path, digest in files] + return {"files": entries, "count": len(entries), "truncated": False} + + +_LOCKFILE_A = ("package-lock.json", "a" * 64) +_LOCKFILE_B = ("go.sum", "b" * 64) +_BASE_INVENTORY = _inventory(_LOCKFILE_A, _LOCKFILE_B) +_BASE_SCANNER_VERSION = "12.3.3" +_BASE_SCAN_CONFIG: dict[str, object] = { + "cdxgen_spec_version": "1.5", + "cdxgen_fetch_license": False, + "scan_scope_filter_enabled": True, + "scan_scope_filter_maven_enabled": True, + "scan_scope_filter_node_enabled": True, +} + + +#: Sentinel distinguishing "caller did not pass this kwarg" (use the base +#: fixture) from "caller explicitly passed None" (a real test case below; +#: None is not the default, it is the value under test). +_UNSET = object() + + +def _compute( + *, + inventory: dict[str, object] | None | object = _UNSET, + scanner_version: str | None = None, + scan_config: dict[str, object] | None = None, +) -> str | None: + resolved_inventory = _BASE_INVENTORY if inventory is _UNSET else inventory + return compute_scan_fingerprint( + manifest_inventory=resolved_inventory, # type: ignore[arg-type] + scanner_version=_BASE_SCANNER_VERSION if scanner_version is None else scanner_version, + scan_config=_BASE_SCAN_CONFIG if scan_config is None else scan_config, + ) + + +# --------------------------------------------------------------------------- +# 1. Determinism (regression contract §4 S8) +# --------------------------------------------------------------------------- + + +def test_same_inputs_produce_the_same_digest_twice() -> None: + first = _compute() + second = _compute() + assert first is not None + assert first == second + + +def test_determinism_is_independent_of_dict_and_list_insertion_order() -> None: + """A caller building the inventory or config dict in a different order + (e.g. a filesystem walk that visits entries differently) must not change + the digest: only the CONTENT may change it. + """ + reordered_inventory = _inventory(_LOCKFILE_B, _LOCKFILE_A) + reordered_config = dict(reversed(list(_BASE_SCAN_CONFIG.items()))) + + baseline = _compute() + reordered = _compute(inventory=reordered_inventory, scan_config=reordered_config) + + assert baseline == reordered + + +def test_does_not_mutate_its_inputs() -> None: + """A pure function must not leave the caller's mappings changed: the + inventory dict is also stored verbatim on the scan row as + ``input_manifests``, and a mutation here would corrupt that record. + """ + inventory_copy = copy.deepcopy(_BASE_INVENTORY) + config_copy = copy.deepcopy(_BASE_SCAN_CONFIG) + + compute_scan_fingerprint( + manifest_inventory=inventory_copy, + scanner_version=_BASE_SCANNER_VERSION, + scan_config=config_copy, + ) + + assert inventory_copy == _BASE_INVENTORY + assert config_copy == _BASE_SCAN_CONFIG + + +# --------------------------------------------------------------------------- +# 2. Scanner version (the plan's explicit accuracy requirement) +# --------------------------------------------------------------------------- + + +def test_scanner_version_bump_changes_the_fingerprint() -> None: + """The load-bearing case: after a worker image upgrade, a reuse decision + built on this fingerprint must NOT serve an SBOM cdxgen 12.3.3 produced + as though cdxgen 12.4.0 would have produced the same bytes. + """ + before = _compute(scanner_version="12.3.3") + after = _compute(scanner_version="12.4.0") + + assert before is not None + assert after is not None + assert before != after + + +def test_scanner_version_is_compared_as_an_exact_string() -> None: + """No implicit semver normalization: "12.3.3" and "12.3.30" must not + collide by being treated as numerically equal or truncated. + """ + a = _compute(scanner_version="12.3.3") + b = _compute(scanner_version="12.3.30") + assert a != b + + +# --------------------------------------------------------------------------- +# 3. Scan config (spec version, scope-filter toggles) +# --------------------------------------------------------------------------- + + +def test_cdxgen_spec_version_change_changes_the_fingerprint() -> None: + before = _compute(scan_config={**_BASE_SCAN_CONFIG, "cdxgen_spec_version": "1.5"}) + after = _compute(scan_config={**_BASE_SCAN_CONFIG, "cdxgen_spec_version": "1.6"}) + assert before != after + + +def test_each_scope_filter_toggle_independently_changes_the_fingerprint() -> None: + baseline = _compute() + for key in ( + "scan_scope_filter_enabled", + "scan_scope_filter_maven_enabled", + "scan_scope_filter_node_enabled", + ): + flipped = dict(_BASE_SCAN_CONFIG) + flipped[key] = not flipped[key] + variant = _compute(scan_config=flipped) + assert variant != baseline, f"flipping {key} did not change the fingerprint" + + +def test_fetch_license_toggle_changes_the_fingerprint() -> None: + before = _compute(scan_config={**_BASE_SCAN_CONFIG, "cdxgen_fetch_license": False}) + after = _compute(scan_config={**_BASE_SCAN_CONFIG, "cdxgen_fetch_license": True}) + assert before != after + + +def test_unexpected_config_value_shapes_are_stringified_not_dropped() -> None: + """A future scan-config key this function was not written against (a + list, a nested dict) must still influence the digest: silently ignoring + it would narrow the fingerprint without anyone deciding that on purpose. + """ + with_list = _compute(scan_config={**_BASE_SCAN_CONFIG, "extra": ["a", "b"]}) + with_different_list = _compute(scan_config={**_BASE_SCAN_CONFIG, "extra": ["a", "c"]}) + without = _compute() + + assert with_list != without + assert with_list != with_different_list + + +# --------------------------------------------------------------------------- +# 4. Lockfile content (the baseline the feature exists to detect) +# --------------------------------------------------------------------------- + + +def test_lockfile_hash_change_changes_the_fingerprint() -> None: + before = _compute() + changed = _inventory(("package-lock.json", "c" * 64), _LOCKFILE_B) + after = _compute(inventory=changed) + assert before != after + + +def test_added_lockfile_changes_the_fingerprint() -> None: + before = _compute(inventory=_inventory(_LOCKFILE_A)) + after = _compute(inventory=_inventory(_LOCKFILE_A, _LOCKFILE_B)) + assert before != after + + +def test_removed_lockfile_changes_the_fingerprint() -> None: + before = _compute(inventory=_inventory(_LOCKFILE_A, _LOCKFILE_B)) + after = _compute(inventory=_inventory(_LOCKFILE_A)) + assert before != after + + +def test_manifest_only_ecosystem_change_is_detected() -> None: + """Maven / a bare requirements.txt has no separate lockfile: the + manifest itself is the authoritative dependency declaration there, which + is why the inventory is not narrowed to "lockfiles only" (see the module + docstring). A pom.xml version bump must still change the digest. + """ + before = _compute(inventory=_inventory(("pom.xml", "d" * 64))) + after = _compute(inventory=_inventory(("pom.xml", "e" * 64))) + assert before != after + + +# --------------------------------------------------------------------------- +# 5. Refusing to answer (None is not a wildcard match) +# --------------------------------------------------------------------------- + + +def test_none_inventory_returns_none() -> None: + """No manifest/lockfile found, or a scan with no source tree (container / + SBOM-ingest): there is no dependency-set identity to fingerprint. + """ + assert _compute(inventory=None) is None + + +def test_truncated_inventory_returns_none() -> None: + """A walk that stopped before covering the tree cannot rule out a change + past the cutoff: treating it as complete would be the exact failure + mode a fingerprint exists to prevent. + """ + truncated = { + "files": [{"path": "package.json", "size": 1, "sha256": "f" * 64}], + "count": 1, + "truncated": True, + } + assert _compute(inventory=truncated) is None + + +def test_unhashed_file_returns_none() -> None: + """A file too large to hash (or unreadable) leaves sha256=None in the + inventory (services.scan_inputs._sha256's contract). "Unknown content" + must not be treated as "unchanged content". + """ + unhashed = { + "files": [{"path": "package-lock.json", "size": 999_999_999, "sha256": None}], + "count": 1, + "truncated": False, + } + assert _compute(inventory=unhashed) is None + + +def test_empty_files_list_returns_none() -> None: + empty = {"files": [], "count": 0, "truncated": False} + assert _compute(inventory=empty) is None + + +def test_non_mapping_file_entry_returns_none() -> None: + """The inventory contract promises a list of dicts; a malformed entry + (a bare string, say) must not crash the caller. Refusing to answer is + the same defensive posture the whole module takes elsewhere. + """ + malformed = {"files": ["not-a-dict"], "count": 1, "truncated": False} + assert _compute(inventory=malformed) is None + + +def test_file_entry_missing_path_returns_none() -> None: + missing_path = { + "files": [{"size": 1, "sha256": "a" * 64}], + "count": 1, + "truncated": False, + } + assert _compute(inventory=missing_path) is None + + +def test_file_entry_with_empty_path_returns_none() -> None: + empty_path = { + "files": [{"path": "", "size": 1, "sha256": "a" * 64}], + "count": 1, + "truncated": False, + } + assert _compute(inventory=empty_path) is None + + +def test_two_none_results_are_not_equal_by_construction() -> None: + """Guards against a future edit that makes this return a sentinel string + instead of None for the "cannot fingerprint" case: a caller comparing + two scans' stored fingerprints with ``==`` must never see two + un-fingerprinted scans read as "the same". + """ + a = _compute(inventory=None) + b = _compute(inventory=None) + assert a is None + assert b is None + assert a == b # both None, which is exactly why callers must not use + # equality alone to decide reuse; None must be special-cased first. The + # assertion documents the trap rather than hiding it. + + +# --------------------------------------------------------------------------- +# Output shape +# --------------------------------------------------------------------------- + + +def test_digest_is_a_64_character_lowercase_hex_string() -> None: + digest = _compute() + assert digest is not None + assert len(digest) == 64 + assert digest == digest.lower() + int(digest, 16) # raises ValueError if not valid hex + + +def test_schema_version_constant_is_hashed_in() -> None: + """A change to FINGERPRINT_SCHEMA_VERSION must change every digest, even + for identical lockfile/scanner/config inputs: otherwise a future + incompatible change to this module's hashing shape could produce a + digest that collides with one an older worker wrote under the old shape. + """ + assert FINGERPRINT_SCHEMA_VERSION == 1 + digest_v1 = _compute() + + import models.scan_fingerprint as fp_module + + original = fp_module.FINGERPRINT_SCHEMA_VERSION + try: + fp_module.FINGERPRINT_SCHEMA_VERSION = 2 # type: ignore[misc] + digest_v2 = _compute() + finally: + fp_module.FINGERPRINT_SCHEMA_VERSION = original # type: ignore[misc] + + assert digest_v1 != digest_v2