diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 000c506..a648827 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -10,6 +10,9 @@ trading research systems. The compatible Python import namespace is | --- | --- | | `backtest_harness.fee_models` | Protocol and implementations for exchange fee calculations. | | `backtest_harness.monte_carlo` | Equity-path sampling and risk percentile estimation. | +| `backtest_harness.analytics` | Tear-sheet statistics and walk-forward index splitting. | +| `backtest_harness.provider_receipts` | Deterministic, privacy-safe evaluation receipts (ADR-021). | +| `backtest_harness.evidence` | Counterfactual provider evidence and Core-compatible export (BACK-001). | ## Data flow @@ -44,6 +47,34 @@ flowchart TD LLM[verdict] --> KT ``` +## Counterfactual evidence (BACK-001) + +`backtest_harness.evidence` exposes deterministic, counterfactual backtest +evaluations as verification evidence compatible with Verdict Core's +`VerificationResult` and `EvidenceChainLink` contracts: + +- **`run_counterfactual`** — Seeded, reproducible evaluation of a historical + returns series. Records inputs, random seed, fee model, code version, and + dataset reference in a receipt whose `evidence_refs` bind it to the canonical + hash of produced results (Monte Carlo, tear-sheet, walk-forward, fees). Pure + in-process math; no network, no live execution. +- **`build_failure_evidence`** — Explicit receipts for failure, timeout, + denied, cancelled, and unknown states. The absence of results is auditable; + fabricated success is impossible because the bundle carries none. +- **`to_verification_result`** — Export as a Core `VerificationResult` payload. + Status derives only from the recorded outcome; a non-success outcome can + never map to `"passed"`, and advisory detail fields cannot override the + mapping. +- **`to_evidence_chain_link`** — Export as a Core `EvidenceChainLink` payload. + All decision-authority fields (decision, policy, envelope hash, model, + timestamp) must be supplied by the caller; this provider is evidence-only and + cannot grant or fabricate authority. + +The receipts implement ADR-021 (Deterministic Provider Evaluation Receipts): +identical evaluation inputs produce identical `inputs_hash` and `config_hash`, +enabling deterministic replay. Sensitive keys (`api_key`, `authorization`, +`password`, `secret`, `token`) are rejected at the receipt boundary. + ## Design principles - **Auditability over cleverness**: Backtest assumptions should be inspectable. diff --git a/src/backtest_harness/__init__.py b/src/backtest_harness/__init__.py index c145b49..5258447 100644 --- a/src/backtest_harness/__init__.py +++ b/src/backtest_harness/__init__.py @@ -27,15 +27,25 @@ """ from backtest_harness.analytics import split_walk_forward, tearsheet +from backtest_harness.evidence import ( + build_failure_evidence, + run_counterfactual, + to_evidence_chain_link, + to_verification_result, +) from backtest_harness.fee_models import ( BoundedProfitFeeModel, FeeModel, FlatMakerTakerModel, ) from backtest_harness.monte_carlo import MonteCarloSimulator -from backtest_harness.provider_receipts import build_backtest_receipt +from backtest_harness.provider_receipts import ( + PROVIDER_VERSION, + build_backtest_receipt, + canonical_hash, +) -__version__ = "0.2.0" +__version__ = PROVIDER_VERSION __all__ = [ "BoundedProfitFeeModel", @@ -43,6 +53,11 @@ "FlatMakerTakerModel", "MonteCarloSimulator", "build_backtest_receipt", + "build_failure_evidence", + "canonical_hash", + "run_counterfactual", "split_walk_forward", "tearsheet", + "to_evidence_chain_link", + "to_verification_result", ] diff --git a/src/backtest_harness/analytics.py b/src/backtest_harness/analytics.py index 133c623..924a569 100644 --- a/src/backtest_harness/analytics.py +++ b/src/backtest_harness/analytics.py @@ -16,7 +16,7 @@ from collections.abc import Iterator from math import sqrt -from typing import Any +from typing import Any, Literal, overload import numpy as np @@ -34,6 +34,14 @@ def _to_returns(returns: Any) -> np.ndarray: return arr +@overload +def tearsheet( + returns: Any, periods_per_year: int = _TRADING_PERIODS, as_dict: Literal[True] = True +) -> dict[str, float | int]: ... +@overload +def tearsheet( + returns: Any, periods_per_year: int = _TRADING_PERIODS, as_dict: Literal[False] = False +) -> None: ... def tearsheet( returns: Any, periods_per_year: int = _TRADING_PERIODS, as_dict: bool = True ) -> dict | None: diff --git a/src/backtest_harness/evidence.py b/src/backtest_harness/evidence.py new file mode 100644 index 0000000..c2ca8ef --- /dev/null +++ b/src/backtest_harness/evidence.py @@ -0,0 +1,446 @@ +"""Deterministic counterfactual evaluation and Core-compatible evidence export. + +This module exposes Monte Carlo, fee, walk-forward, and tear-sheet results as +verification / counterfactual evidence (BACK-001, ADR-021): + +* :func:`run_counterfactual` — seeded, reproducible evaluation of a historical + returns series. Pure in-process math: no network, no live execution, no + order placement. +* :func:`build_failure_evidence` — explicit receipts for failure, timeout, + denial, unavailable, and unknown states so an absent result is never read as + a pass. +* :func:`to_verification_result` — export as a verdict-core + ``VerificationResult`` payload. Status derives only from the recorded + outcome; a non-success outcome can never map to ``"passed"``, and advisory + detail fields cannot override the mapping. +* :func:`to_evidence_chain_link` — export as a verdict-core + ``EvidenceChainLink`` payload. All decision-authority fields (decision, + policy, envelope hash, model, timestamp) MUST be supplied by the caller; + this provider is evidence-only and cannot grant or fabricate authority. +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping, Sequence +from typing import Any + +import numpy as np + +from backtest_harness.analytics import split_walk_forward, tearsheet +from backtest_harness.fee_models import BoundedProfitFeeModel, FlatMakerTakerModel +from backtest_harness.monte_carlo import MonteCarloSimulator +from backtest_harness.provider_receipts import ( + PROVIDER_VERSION, + build_backtest_receipt, + canonical_hash, +) + +EVIDENCE_SCHEMA_VERSION = "1" +PROVIDER_NAME = "verdict-backtest" + +# Outcome allowlist mirroring verdict-core ``_OUTCOME_VALUES``. Failure, +# timeout, denial, and unknown states are explicit, never inferred. +COUNTERFACTUAL_OUTCOMES = frozenset( + { + "success", + "failure", + "partial", + "denied", + "unknown", + "cancelled", + "timeout", + "error", + "skipped", + } +) + +# Deterministic outcome -> VerificationResult.status mapping. ``"passed"`` +# appears exactly once: only a recorded ``"success"`` can produce it, so no +# advisory/provider data can weaken a hard policy gate by upgrading a +# non-success run. Inconclusive states stay ``"unknown"`` (verdict-core keeps +# ``unknown`` distinct from ``passed`` for the same reason). +_OUTCOME_TO_STATUS: Mapping[str, str] = { + "success": "passed", + "failure": "failed", + "denied": "failed", + "error": "failed", + "partial": "unknown", + "timeout": "unknown", + "unknown": "unknown", + "cancelled": "skipped", + "skipped": "skipped", +} + +# Mirrors verdict-core's digest and ISO-8601 timestamp patterns so exports +# fail loudly at this boundary even when verdict-core is not installed. +_DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") +_ISO_TIMESTAMP = re.compile( + r"^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:?\d{2})?$" +) + +_FEE_MODELS = ("bounded_profit", "flat_maker_taker") + + +def run_counterfactual( + *, + run_id: str, + trade_returns: Sequence[float], + starting_equity: float, + seed: int, + dataset_ref: str, + num_simulations: int = 1000, + trades_per_sim: int = 250, + walk_forward_splits: int = 5, + periods_per_year: int = 252, + fee_config: Mapping[str, Any] | None = None, + fee_trades: Sequence[tuple[float, Any]] | None = None, + provenance: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Run a seeded, reproducible counterfactual backtest evaluation. + + Records inputs, the random seed, the fee model, the code version, and the + dataset reference in a receipt whose ``evidence_refs`` bind it to the + canonical hash of the produced results. Identical arguments produce an + identical evidence bundle. Pure in-process computation only. + + Args: + run_id: Unique identifier for this evaluation run. + trade_returns: Historical per-trade returns as fractions. + starting_equity: Initial equity in base currency units. + seed: Random seed for the Monte Carlo resampling (recorded). + dataset_ref: Caller-supplied reference identifying the input dataset. + num_simulations: Monte Carlo path count. + trades_per_sim: Trades per simulated path. + walk_forward_splits: Number of walk-forward blocks (>= 2). + periods_per_year: Annualization factor for tear-sheet statistics. + fee_config: Optional fee model description, e.g. + ``{"model": "bounded_profit", "percent_of_profit": 0.07, + "maximum_fee_cents": 0.05}`` or + ``{"model": "flat_maker_taker", "maker_bps": 0.0, + "taker_bps": 0.001}``. + fee_trades: Trades evaluated under ``fee_config``: + ``(entry_cents, payout_cents)`` pairs for ``bounded_profit`` or + ``(trade_volume, is_maker)`` pairs for ``flat_maker_taker``. + provenance: Optional redacted provenance mapping for the receipt. + + Returns: + Evidence bundle ``{"schema_version", "receipt", "results", + "results_hash"}``. + + Raises: + ValueError: On malformed input (empty/NaN returns, non-positive + equity or simulation counts, unknown fee model, bad splits). + """ + returns = _validate_returns(trade_returns) + if not run_id.strip(): + raise ValueError("run_id must be non-empty") + if not dataset_ref.strip(): + raise ValueError("dataset_ref must be non-empty") + if starting_equity <= 0: + raise ValueError("starting_equity must be positive") + if num_simulations <= 0 or trades_per_sim <= 0: + raise ValueError("num_simulations and trades_per_sim must be positive") + if not 2 <= walk_forward_splits <= returns.size: + raise ValueError("walk_forward_splits must be >= 2 and <= the number of return periods") + fee_summary = _evaluate_fees(fee_config, fee_trades) + + monte_carlo = _seeded_monte_carlo( + returns=returns, + starting_equity=starting_equity, + seed=seed, + num_simulations=num_simulations, + trades_per_sim=trades_per_sim, + ) + sheet = tearsheet(returns, periods_per_year=periods_per_year) + walk_forward = _walk_forward_report(returns, walk_forward_splits, periods_per_year) + + results: dict[str, Any] = { + "monte_carlo": monte_carlo, + "tearsheet": sheet, + "walk_forward": walk_forward, + "fees": fee_summary, + } + results_hash = canonical_hash(results) + inputs = { + "dataset_ref": dataset_ref, + "trade_returns_hash": canonical_hash([float(v) for v in returns]), + "n_periods": int(returns.size), + } + config = { + "seed": int(seed), + "starting_equity": float(starting_equity), + "num_simulations": int(num_simulations), + "trades_per_sim": int(trades_per_sim), + "walk_forward_splits": int(walk_forward_splits), + "periods_per_year": int(periods_per_year), + "fee_config": dict(fee_config) if fee_config is not None else None, + "code_version": PROVIDER_VERSION, + } + receipt = build_backtest_receipt( + run_id=run_id, + inputs=inputs, + config=config, + outcome="success", + evidence_refs=(results_hash,), + provenance=provenance, + details={"kind": "counterfactual_backtest"}, + ) + return { + "schema_version": EVIDENCE_SCHEMA_VERSION, + "receipt": receipt, + "results": results, + "results_hash": results_hash, + } + + +def build_failure_evidence( + *, + run_id: str, + outcome: str, + reason: str, + dataset_ref: str = "", + config: Mapping[str, Any] | None = None, + provenance: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Record an explicit non-success state as an evidence bundle. + + Use for failure, timeout, denied, cancelled, or unknown states so the + absence of results is itself auditable. ``outcome`` must be a + non-success member of the outcome allowlist; fabricated results are + impossible because the bundle carries none. + """ + if outcome not in COUNTERFACTUAL_OUTCOMES: + raise ValueError(f"unknown outcome: {outcome}") + if outcome == "success": + raise ValueError("success evidence must come from run_counterfactual") + if not reason.strip(): + raise ValueError("reason must be non-empty") + receipt = build_backtest_receipt( + run_id=run_id, + inputs={"dataset_ref": dataset_ref}, + config={**dict(config or {}), "code_version": PROVIDER_VERSION}, + outcome=outcome, + provenance=provenance, + details={"kind": "counterfactual_backtest", "reason": reason}, + ) + return { + "schema_version": EVIDENCE_SCHEMA_VERSION, + "receipt": receipt, + "results": None, + "results_hash": None, + } + + +def to_verification_result( + evidence: Mapping[str, Any], + *, + check_name: str = "backtest_counterfactual", + policy_requirement: str = "", + command: str = "", + duration_ms: int | None = None, +) -> dict[str, Any]: + """Export an evidence bundle as a Core ``VerificationResult`` payload. + + ``status`` derives solely from the receipt outcome via the fixed + non-upgradable mapping; advisory fields in ``details`` or the receipt + cannot override it. + """ + receipt = _require_receipt(evidence) + outcome = receipt["outcome"] + if outcome not in _OUTCOME_TO_STATUS: + raise ValueError(f"unknown outcome: {outcome}") + digests = [receipt["inputs_hash"], receipt["config_hash"]] + results_hash = evidence.get("results_hash") + if results_hash is not None: + digests.append(results_hash) + for digest in digests: + if not _DIGEST_PATTERN.match(str(digest)): + raise ValueError(f"invalid artifact digest: {digest}") + # Core contracts reject nested nulls in generic payload maps, so absent + # values are omitted rather than carried as ``None``. + details: dict[str, Any] = {"receipt": dict(receipt)} + if results_hash is not None: + details["results_hash"] = results_hash + payload: dict[str, Any] = { + "check_name": check_name, + "check_type": "custom", + "status": _OUTCOME_TO_STATUS[outcome], + "details": details, + "artifact_digests": digests, + "command": command, + "runtime": f"backtest_harness=={PROVIDER_VERSION}", + "provenance": PROVIDER_NAME, + "policy_requirement": policy_requirement, + "raw_output": "", + "schema_version": "1", + } + if duration_ms is not None: + payload["duration_ms"] = duration_ms + return payload + + +def to_evidence_chain_link( + evidence: Mapping[str, Any], + *, + decision: str, + policy: str, + envelope_hash: str, + runtime: str, + model: str, + timestamp: str, + previous_hash: str = "", + tools: Sequence[str] = (), + changes: Sequence[str] = (), +) -> dict[str, Any]: + """Export an evidence bundle as a Core ``EvidenceChainLink`` payload. + + Every decision-authority field is caller-supplied and validated; this + provider only contributes the verification payload and its outcome. It + cannot mint decisions, policies, or envelopes. + """ + receipt = _require_receipt(evidence) + for name, value in ( + ("decision", decision), + ("policy", policy), + ("runtime", runtime), + ("model", model), + ): + if not value.strip(): + raise ValueError(f"{name} must not be empty") + if not _DIGEST_PATTERN.match(envelope_hash): + raise ValueError(f"invalid envelope_hash: {envelope_hash}") + if previous_hash and not _DIGEST_PATTERN.match(previous_hash): + raise ValueError(f"invalid previous_hash: {previous_hash}") + if not _ISO_TIMESTAMP.match(timestamp): + raise ValueError(f"invalid timestamp: {timestamp}") + outcome = receipt["outcome"] + if outcome not in COUNTERFACTUAL_OUTCOMES: + raise ValueError(f"unknown outcome: {outcome}") + return { + "decision": decision, + "policy": policy, + "envelope_hash": envelope_hash, + "runtime": runtime, + "provider": PROVIDER_NAME, + "model": model, + "tools": list(tools), + "changes": list(changes), + "verification": [to_verification_result(evidence)], + "outcome": outcome, + "timestamp": timestamp, + "previous_hash": previous_hash, + "schema_version": "1", + } + + +def _validate_returns(trade_returns: Sequence[float]) -> np.ndarray: + try: + returns = np.asarray(trade_returns, dtype=np.float64).ravel() + except (TypeError, ValueError) as exc: + raise ValueError(f"trade_returns must be numeric: {exc}") from exc + if returns.size == 0: + raise ValueError("trade_returns must not be empty") + if not np.all(np.isfinite(returns)): + raise ValueError("trade_returns must contain only finite values") + return returns + + +def _seeded_monte_carlo( + *, + returns: np.ndarray, + starting_equity: float, + seed: int, + num_simulations: int, + trades_per_sim: int, +) -> dict[str, Any]: + """Run the simulator under a saved/restored seeded global RNG state.""" + state = np.random.get_state() + try: + np.random.seed(seed) + return MonteCarloSimulator.simulate_equity_paths( + trade_returns_pct=returns, + starting_equity=starting_equity, + num_simulations=num_simulations, + trades_per_sim=trades_per_sim, + ) + finally: + np.random.set_state(state) + + +def _walk_forward_report( + returns: np.ndarray, n_splits: int, periods_per_year: int +) -> list[dict[str, Any]]: + folds: list[dict[str, Any]] = [] + for fold, (train_idx, test_idx) in enumerate( + split_walk_forward(returns, n_splits=n_splits), start=1 + ): + test_sheet = tearsheet(returns[test_idx], periods_per_year=periods_per_year) + folds.append( + { + "fold": fold, + "train_size": int(train_idx.size), + "test_size": int(test_idx.size), + "test_total_return": test_sheet["total_return"], + "test_sharpe": test_sheet["sharpe"], + "test_max_drawdown": test_sheet["max_drawdown"], + } + ) + return folds + + +def _evaluate_fees( + fee_config: Mapping[str, Any] | None, + fee_trades: Sequence[tuple[float, Any]] | None, +) -> dict[str, Any] | None: + if fee_config is None: + if fee_trades: + raise ValueError("fee_trades requires fee_config") + return None + config = dict(fee_config) + model_name = config.pop("model", None) + if model_name not in _FEE_MODELS: + raise ValueError(f"fee_config.model must be one of {sorted(_FEE_MODELS)}") + trades = list(fee_trades or ()) + try: + if model_name == "bounded_profit": + bounded = BoundedProfitFeeModel(**config) + fees = [bounded.calculate_fee(float(entry), float(payout)) for entry, payout in trades] + else: + flat = FlatMakerTakerModel(**config) + fees = [ + flat.calculate_fee(float(volume), bool(is_maker)) for volume, is_maker in trades + ] + except TypeError as exc: + raise ValueError(f"malformed fee_config or fee_trades: {exc}") from exc + total = float(sum(fees)) + return { + "model": model_name, + "params": {key: float(value) for key, value in config.items()}, + "n_trades": len(fees), + "total_fee": total, + "mean_fee": total / len(fees) if fees else 0.0, + } + + +def _require_receipt(evidence: Mapping[str, Any]) -> Mapping[str, Any]: + if not isinstance(evidence, Mapping): + raise ValueError("evidence must be a mapping") + receipt = evidence.get("receipt") + if not isinstance(receipt, Mapping): + raise ValueError("evidence.receipt must be a mapping") + for field in ("outcome", "inputs_hash", "config_hash"): + if field not in receipt: + raise ValueError(f"evidence.receipt missing field: {field}") + return receipt + + +__all__ = [ + "COUNTERFACTUAL_OUTCOMES", + "EVIDENCE_SCHEMA_VERSION", + "PROVIDER_NAME", + "build_failure_evidence", + "run_counterfactual", + "to_evidence_chain_link", + "to_verification_result", +] diff --git a/src/backtest_harness/provider_receipts.py b/src/backtest_harness/provider_receipts.py index 39ab312..82987d1 100644 --- a/src/backtest_harness/provider_receipts.py +++ b/src/backtest_harness/provider_receipts.py @@ -9,6 +9,22 @@ SCHEMA_VERSION = "1" +# Single source of truth for the provider version stamped into receipts and +# re-exported as ``backtest_harness.__version__``. +PROVIDER_VERSION = "0.3.0" + + +def canonical_hash(value: Any) -> str: + """Hash JSON-compatible input using stable canonical serialization. + + Matches verdict-core's ``canonical_hash`` (ADR-021) byte-for-byte so that + identical payloads hashed by either side produce identical digests. + Non-JSON-compatible input raises ``TypeError`` instead of being silently + coerced. + """ + encoded = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return f"sha256:{hashlib.sha256(encoded.encode('utf-8')).hexdigest()}" + def build_backtest_receipt( *, @@ -29,7 +45,7 @@ def build_backtest_receipt( "schema_version": SCHEMA_VERSION, "run_id": run_id, "provider": "verdict-backtest", - "provider_version": "0.2.0", + "provider_version": PROVIDER_VERSION, "inputs_hash": _hash(inputs), "config_hash": _hash(config), "outcome": outcome, @@ -61,4 +77,9 @@ def _reject_sensitive(value: Any) -> None: _reject_sensitive(child) -__all__ = ["SCHEMA_VERSION", "build_backtest_receipt"] +__all__ = [ + "PROVIDER_VERSION", + "SCHEMA_VERSION", + "build_backtest_receipt", + "canonical_hash", +] diff --git a/tests/test_core_conformance.py b/tests/test_core_conformance.py new file mode 100644 index 0000000..fd8f7fd --- /dev/null +++ b/tests/test_core_conformance.py @@ -0,0 +1,76 @@ +"""Boundary conformance against real verdict-core contracts (ADR-021, BACK-001). + +Runs only when ``verdict`` (verdict-core) is installed — CI executes it in the +``compat-gate`` job. Exercises the exported payloads through Core's own +``ProviderReceipt``, ``VerificationResult``, and ``EvidenceChainLink`` +validators so drift fails here rather than at replay time. +""" + +from __future__ import annotations + +import pytest + +verdict_contracts = pytest.importorskip("verdict.contracts") +verdict_receipts = pytest.importorskip("verdict.provider_receipts") + +from backtest_harness.evidence import ( # noqa: E402 + build_failure_evidence, + run_counterfactual, + to_evidence_chain_link, + to_verification_result, +) +from backtest_harness.provider_receipts import canonical_hash # noqa: E402 + + +def _evidence(): + return run_counterfactual( + run_id="conformance-1", + trade_returns=[0.02, -0.01, 0.03, 0.005, -0.02, 0.015, 0.01, -0.005], + starting_equity=1000.0, + seed=7, + dataset_ref="dataset://fixtures/returns-v1", + num_simulations=100, + trades_per_sim=25, + walk_forward_splits=4, + fee_config={"model": "bounded_profit"}, + fee_trades=[(50.0, 100.0)], + ) + + +def test_canonical_hash_matches_core() -> None: + payload = {"b": [1, 2.5, "x"], "a": {"nested": True, "n": None}} + assert canonical_hash(payload) == verdict_receipts.canonical_hash(payload) + + +def test_receipt_round_trips_through_core_provider_receipt() -> None: + receipt = _evidence()["receipt"] + core_receipt = verdict_receipts.ProviderReceipt.from_dict(receipt) + assert core_receipt.to_dict() == receipt + + +def test_verification_result_export_passes_core_contract() -> None: + evidence = _evidence() + payload = to_verification_result(evidence, policy_requirement="BACK-001") + result = verdict_contracts.VerificationResult.from_dict(payload) + assert result.status == "passed" + assert result.check_type == "custom" + + failed = build_failure_evidence(run_id="conformance-2", outcome="failure", reason="x") + failed_result = verdict_contracts.VerificationResult.from_dict(to_verification_result(failed)) + assert failed_result.status == "failed" + + +def test_evidence_chain_link_export_passes_core_contract() -> None: + evidence = _evidence() + link_payload = to_evidence_chain_link( + evidence, + decision="allow", + policy="policy://backtest/counterfactual", + envelope_hash=canonical_hash({"envelope": "conformance"}), + runtime="verdict-node", + model="none", + timestamp="2026-08-20T00:00:00Z", + ) + link = verdict_contracts.EvidenceChainLink.from_dict(link_payload) + assert link.provider == "verdict-backtest" + assert link.outcome == "success" diff --git a/tests/test_evidence.py b/tests/test_evidence.py new file mode 100644 index 0000000..862d226 --- /dev/null +++ b/tests/test_evidence.py @@ -0,0 +1,236 @@ +"""Counterfactual provider and evidence-export tests (BACK-001).""" + +from __future__ import annotations + +import copy +import json + +import numpy as np +import pytest + +from backtest_harness.evidence import ( + COUNTERFACTUAL_OUTCOMES, + build_failure_evidence, + run_counterfactual, + to_evidence_chain_link, + to_verification_result, +) +from backtest_harness.provider_receipts import canonical_hash + +RETURNS = [0.02, -0.01, 0.03, 0.005, -0.02, 0.015, 0.01, -0.005, 0.02, -0.01] +ENVELOPE_HASH = canonical_hash({"envelope": "test"}) +TIMESTAMP = "2026-08-20T00:00:00Z" + + +def _run(**overrides): + kwargs = { + "run_id": "run-1", + "trade_returns": RETURNS, + "starting_equity": 1000.0, + "seed": 42, + "dataset_ref": "dataset://fixtures/returns-v1", + "num_simulations": 200, + "trades_per_sim": 50, + "walk_forward_splits": 5, + } + kwargs.update(overrides) + return run_counterfactual(**kwargs) + + +class TestRunCounterfactual: + def test_same_seed_reproduces_identical_evidence(self): + first = _run() + second = _run() + assert first == second + assert first["results_hash"] == second["results_hash"] + assert first["receipt"]["inputs_hash"] == second["receipt"]["inputs_hash"] + assert first["receipt"]["config_hash"] == second["receipt"]["config_hash"] + + def test_different_seed_changes_results(self): + assert _run(seed=1)["results_hash"] != _run(seed=2)["results_hash"] + + def test_records_seed_fee_model_code_version_and_dataset(self): + evidence = _run( + fee_config={ + "model": "bounded_profit", + "percent_of_profit": 0.07, + "maximum_fee_cents": 0.05, + }, + fee_trades=[(50.0, 100.0), (60.0, 0.0)], + ) + receipt = evidence["receipt"] + # The receipt binds the results by canonical hash. + assert evidence["results_hash"] in receipt["evidence_refs"] + assert evidence["results_hash"] == canonical_hash(evidence["results"]) + # Bundle is JSON-compatible (portable evidence). + json.dumps(evidence) + results = evidence["results"] + assert set(results) == {"monte_carlo", "tearsheet", "walk_forward", "fees"} + assert results["fees"] == { + "model": "bounded_profit", + "params": {"percent_of_profit": 0.07, "maximum_fee_cents": 0.05}, + "n_trades": 2, + "total_fee": pytest.approx(0.05), # capped fee + zero on a loss + "mean_fee": pytest.approx(0.025), + } + assert len(results["walk_forward"]) == 4 # n_splits - 1 expanding folds + + def test_flat_maker_taker_fee_summary(self): + evidence = _run( + fee_config={"model": "flat_maker_taker", "maker_bps": 0.0, "taker_bps": 0.001}, + fee_trades=[(1000.0, False), (1000.0, True)], + ) + fees = evidence["results"]["fees"] + assert fees["total_fee"] == pytest.approx(1.0) + assert fees["n_trades"] == 2 + + def test_does_not_leak_global_rng_state(self): + np.random.seed(7) + expected = np.random.random() + np.random.seed(7) + _run() + assert np.random.random() == expected + + @pytest.mark.parametrize( + ("overrides", "match"), + [ + ({"trade_returns": []}, "empty"), + ({"trade_returns": [0.01, float("nan")]}, "finite"), + ({"trade_returns": ["not-a-number"]}, "numeric"), + ({"run_id": " "}, "run_id"), + ({"dataset_ref": ""}, "dataset_ref"), + ({"starting_equity": 0.0}, "starting_equity"), + ({"num_simulations": 0}, "positive"), + ({"walk_forward_splits": 1}, "walk_forward_splits"), + ({"walk_forward_splits": 99}, "walk_forward_splits"), + ({"fee_trades": [(1.0, 2.0)]}, "fee_config"), + ({"fee_config": {"model": "mystery"}}, "fee_config.model"), + ({"fee_config": {"model": "bounded_profit", "bogus": 1.0}}, "malformed"), + ], + ) + def test_malformed_inputs_are_rejected(self, overrides, match): + with pytest.raises(ValueError, match=match): + _run(**overrides) + + def test_sensitive_provenance_is_rejected(self): + with pytest.raises(ValueError, match="sensitive"): + _run(provenance={"api_key": "must-not-persist"}) + + +class TestFailureEvidence: + @pytest.mark.parametrize( + "outcome", ["failure", "timeout", "denied", "unknown", "cancelled", "error"] + ) + def test_non_success_states_are_explicit(self, outcome): + evidence = build_failure_evidence( + run_id="run-1", outcome=outcome, reason="provider unavailable" + ) + assert evidence["receipt"]["outcome"] == outcome + assert evidence["results"] is None + assert evidence["results_hash"] is None + + def test_success_cannot_be_fabricated(self): + with pytest.raises(ValueError, match="run_counterfactual"): + build_failure_evidence(run_id="run-1", outcome="success", reason="nope") + + def test_unknown_outcome_and_empty_reason_rejected(self): + with pytest.raises(ValueError, match="unknown outcome"): + build_failure_evidence(run_id="run-1", outcome="mystery", reason="x") + with pytest.raises(ValueError, match="reason"): + build_failure_evidence(run_id="run-1", outcome="failure", reason=" ") + + +class TestVerificationResultExport: + def test_success_maps_to_passed_with_bound_digests(self): + evidence = _run() + payload = to_verification_result(evidence, policy_requirement="BACK-001") + assert payload["status"] == "passed" + assert payload["check_type"] == "custom" + assert payload["provenance"] == "verdict-backtest" + assert evidence["results_hash"] in payload["artifact_digests"] + assert all(d.startswith("sha256:") for d in payload["artifact_digests"]) + + @pytest.mark.parametrize( + ("outcome", "status"), + [ + ("failure", "failed"), + ("denied", "failed"), + ("error", "failed"), + ("timeout", "unknown"), + ("unknown", "unknown"), + ("cancelled", "skipped"), + ], + ) + def test_non_success_never_maps_to_passed(self, outcome, status): + evidence = build_failure_evidence(run_id="run-1", outcome=outcome, reason="x") + assert to_verification_result(evidence)["status"] == status + + def test_advisory_details_cannot_upgrade_status(self): + # A tampered receipt that *claims* success in advisory fields still + # exports from the recorded outcome, so provider data cannot weaken + # a hard policy gate. + evidence = copy.deepcopy( + build_failure_evidence(run_id="run-1", outcome="failure", reason="x") + ) + evidence["receipt"]["details"]["status"] = "passed" + evidence["receipt"]["details"]["approved"] = True + assert to_verification_result(evidence)["status"] == "failed" + + def test_outcome_outside_allowlist_is_rejected(self): + evidence = copy.deepcopy(_run()) + evidence["receipt"]["outcome"] = "totally-approved" + with pytest.raises(ValueError, match="unknown outcome"): + to_verification_result(evidence) + + def test_tampered_digest_is_rejected(self): + evidence = copy.deepcopy(_run()) + evidence["results_hash"] = "sha256:not-a-digest" + with pytest.raises(ValueError, match="digest"): + to_verification_result(evidence) + + def test_malformed_evidence_is_rejected(self): + with pytest.raises(ValueError, match="mapping"): + to_verification_result("not-a-mapping") # type: ignore[arg-type] + with pytest.raises(ValueError, match="receipt"): + to_verification_result({"results": {}}) + + +class TestEvidenceChainExport: + def _link(self, evidence, **overrides): + kwargs = { + "decision": "allow", + "policy": "policy://backtest/counterfactual", + "envelope_hash": ENVELOPE_HASH, + "runtime": "verdict-node", + "model": "none", + "timestamp": TIMESTAMP, + } + kwargs.update(overrides) + return to_evidence_chain_link(evidence, **kwargs) + + def test_link_embeds_verification_and_outcome(self): + evidence = _run() + link = self._link(evidence) + assert link["provider"] == "verdict-backtest" + assert link["outcome"] == "success" + assert link["verification"] == [to_verification_result(evidence)] + assert link["outcome"] in COUNTERFACTUAL_OUTCOMES + + def test_provider_cannot_mint_decision_authority(self): + evidence = _run() + with pytest.raises(ValueError, match="decision"): + self._link(evidence, decision=" ") + with pytest.raises(ValueError, match="policy"): + self._link(evidence, policy="") + with pytest.raises(ValueError, match="envelope_hash"): + self._link(evidence, envelope_hash="sha256:short") + with pytest.raises(ValueError, match="previous_hash"): + self._link(evidence, previous_hash="bogus") + with pytest.raises(ValueError, match="timestamp"): + self._link(evidence, timestamp="yesterday") + + def test_failure_link_carries_failed_verification(self): + evidence = build_failure_evidence(run_id="run-1", outcome="timeout", reason="x") + link = self._link(evidence) + assert link["outcome"] == "timeout" + assert link["verification"][0]["status"] == "unknown" diff --git a/tests/test_provider_receipts.py b/tests/test_provider_receipts.py new file mode 100644 index 0000000..6b5ee95 --- /dev/null +++ b/tests/test_provider_receipts.py @@ -0,0 +1,73 @@ +"""Receipt contract tests mirroring verdict-core's conformance template (ADR-021).""" + +from __future__ import annotations + +import pytest + +from backtest_harness.provider_receipts import ( + PROVIDER_VERSION, + SCHEMA_VERSION, + build_backtest_receipt, + canonical_hash, +) + + +def test_receipt_is_deterministic_and_hashes_inputs() -> None: + receipt = build_backtest_receipt( + run_id="run-1", + inputs={"value": 1, "name": "sample"}, + config={"threshold": 0.5}, + outcome="success", + provenance={"source": "fixture", "authority": "observed"}, + evidence_refs=("evidence-1",), + details={"approved": True}, + ) + same = build_backtest_receipt( + run_id="run-1", + inputs={"name": "sample", "value": 1}, + config={"threshold": 0.5}, + outcome="success", + provenance={"source": "fixture", "authority": "observed"}, + evidence_refs=("evidence-1",), + details={"approved": True}, + ) + + assert receipt == same + assert receipt["inputs_hash"].startswith("sha256:") + assert receipt["config_hash"].startswith("sha256:") + assert receipt["schema_version"] == SCHEMA_VERSION + assert receipt["provider"] == "verdict-backtest" + assert receipt["provider_version"] == PROVIDER_VERSION + + +def test_receipt_rejects_sensitive_metadata() -> None: + with pytest.raises(ValueError, match="sensitive"): + build_backtest_receipt( + run_id="run-1", + inputs={}, + config={}, + outcome="unknown", + provenance={"api_key": "must-not-persist"}, + ) + with pytest.raises(ValueError, match="sensitive"): + build_backtest_receipt( + run_id="run-1", + inputs={}, + config={}, + outcome="unknown", + details={"nested": [{"Token": "x"}]}, + ) + + +def test_receipt_rejects_empty_identifiers() -> None: + with pytest.raises(ValueError, match="non-empty"): + build_backtest_receipt(run_id=" ", inputs={}, config={}, outcome="success") + with pytest.raises(ValueError, match="non-empty"): + build_backtest_receipt(run_id="run-1", inputs={}, config={}, outcome="") + + +def test_canonical_hash_is_order_invariant_and_strict() -> None: + assert canonical_hash({"a": 1, "b": 2}) == canonical_hash({"b": 2, "a": 1}) + assert canonical_hash([1.5, "x"]).startswith("sha256:") + with pytest.raises(TypeError): + canonical_hash(object())