diff --git a/core/tests/test_release_fleet_executor.py b/core/tests/test_release_fleet_executor.py index 0793d3de..bd5f0bdb 100644 --- a/core/tests/test_release_fleet_executor.py +++ b/core/tests/test_release_fleet_executor.py @@ -1034,6 +1034,245 @@ def bridge_to_foundation( ] +def test_transient_network_classifiers_match_only_momentary_network_loss() -> None: + assert executor._transient_network_error("Could not resolve host: github.com") + assert executor._transient_network_error( + "fatal: unable to access 'https://github.com/': Connection reset by peer" + ) + assert not executor._transient_network_error("fixture lifecycle runtime timed out") + assert not executor._transient_network_error("foundation bridge approval was not exact") + assert executor._transient_network_delivery( + {"status": "not-delivered", "evidence": {"reason": "network-unavailable"}} + ) + assert not executor._transient_network_delivery( + {"status": "not-delivered", "evidence": {"reason": "evidence-invalid"}} + ) + assert not executor._transient_network_delivery({"status": "not-delivered"}) + + +def test_transient_network_follow_up_delivery_retries_and_records_attempts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_repo, source_commit = _executor_source_commit(tmp_path) + slept: list[float] = [] + monkeypatch.setattr(executor.time, "sleep", slept.append) + + class BlippedDeliveryRuntime(_Runtime): + delivery_attempts = 0 + + def deliver_latest_release(self, vault: Path) -> dict[str, object]: + self.delivery_attempts += 1 + if self.delivery_attempts < 3: + return { + "status": "not-delivered", + "evidence": {"reason": "network-unavailable"}, + } + return super().deliver_latest_release(vault) + + runtime = BlippedDeliveryRuntime(_identity("1.81.0", "b")) + run = _execute( + tmp_path, + runtime, + source_repo=source_repo, + source_commit=source_commit, + ) + + assert run.case["reached_follow_up"] is True + assert runtime.delivery_attempts == 3 + assert slept == list(executor._TRANSIENT_DELIVERY_BACKOFF_SECONDS) + transcript = json.loads( + (tmp_path / "case.evidence" / "journey-transcript.json").read_text() + ) + events = {event["id"]: event for event in transcript["events"]} + assert events["foundation-preview"]["journey_transport"] == {"attempt_count": 3} + assert events["bridge-foundation"]["journey_transport"] == {"attempt_count": 1} + assert not ( + tmp_path / "case.evidence" / "follow-up-delivery-failure.diagnostic.json" + ).exists() + + +def test_transient_network_delivery_error_is_retried_without_a_diagnostic( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_repo, source_commit = _executor_source_commit(tmp_path) + monkeypatch.setattr(executor.time, "sleep", lambda _delay: None) + + class DnsBlippedDeliveryRuntime(_Runtime): + delivery_attempts = 0 + + def deliver_latest_release(self, vault: Path) -> dict[str, object]: + self.delivery_attempts += 1 + if self.delivery_attempts == 1: + raise executor.ExecutorError( + "fatal: unable to access 'https://github.com/davekilleen/Dex.git/':" + " Could not resolve host: github.com" + ) + return super().deliver_latest_release(vault) + + runtime = DnsBlippedDeliveryRuntime(_identity("1.81.0", "b")) + run = _execute( + tmp_path, + runtime, + source_repo=source_repo, + source_commit=source_commit, + ) + + assert run.case["reached_follow_up"] is True + assert runtime.delivery_attempts == 2 + transcript = json.loads( + (tmp_path / "case.evidence" / "journey-transcript.json").read_text() + ) + events = {event["id"]: event for event in transcript["events"]} + assert events["foundation-preview"]["journey_transport"] == {"attempt_count": 2} + + +def test_non_network_delivery_failure_is_never_retried( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_repo, source_commit = _executor_source_commit(tmp_path) + monkeypatch.setattr( + executor.time, + "sleep", + lambda _delay: pytest.fail("a non-network delivery failure must not back off"), + ) + + class BrokenDeliveryRuntime(_Runtime): + def deliver_latest_release(self, vault: Path) -> dict[str, object]: + self.calls.append("deliver_latest_release") + return { + "status": "not-delivered", + "evidence": {"reason": "evidence-invalid"}, + } + + runtime = BrokenDeliveryRuntime(_identity("1.81.0", "b")) + with pytest.raises(executor.ExecutorError, match="did not prove"): + _execute( + tmp_path, + runtime, + source_repo=source_repo, + source_commit=source_commit, + ) + + assert runtime.calls.count("deliver_latest_release") == 1 + diagnostic = json.loads( + ( + tmp_path / "case.evidence" / "follow-up-delivery-failure.diagnostic.json" + ).read_text() + ) + assert diagnostic["delivery"]["failure_reason"] == "evidence-invalid" + assert diagnostic["delivery"]["attempt_count"] == 1 + + +def test_transient_network_delivery_still_fails_after_bounded_attempts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_repo, source_commit = _executor_source_commit(tmp_path) + slept: list[float] = [] + monkeypatch.setattr(executor.time, "sleep", slept.append) + + class OfflineDeliveryRuntime(_Runtime): + def deliver_latest_release(self, vault: Path) -> dict[str, object]: + self.calls.append("deliver_latest_release") + return { + "status": "not-delivered", + "evidence": {"reason": "network-unavailable"}, + } + + runtime = OfflineDeliveryRuntime(_identity("1.81.0", "b")) + with pytest.raises(executor.ExecutorError, match="did not prove"): + _execute( + tmp_path, + runtime, + source_repo=source_repo, + source_commit=source_commit, + ) + + assert runtime.calls.count("deliver_latest_release") == 3 + assert slept == list(executor._TRANSIENT_DELIVERY_BACKOFF_SECONDS) + diagnostic = json.loads( + ( + tmp_path / "case.evidence" / "follow-up-delivery-failure.diagnostic.json" + ).read_text() + ) + assert diagnostic["delivery"]["failure_reason"] == "network-unavailable" + assert diagnostic["delivery"]["attempt_count"] == 3 + + +def test_transient_network_bridge_failure_retries_with_fresh_exact_approvals( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_repo, source_commit = _executor_source_commit(tmp_path) + monkeypatch.setattr(executor.time, "sleep", lambda _delay: None) + + class BlippedBridgeRuntime(_Runtime): + def bridge_to_foundation(self, vault: Path, foundation, *, input_fn, output_fn): + if self.calls.count("bridge") == 0: + self.calls.append("bridge") + assert input_fn("topology approval") == "APPLY" + raise executor.ExecutorError( + "curl: (6) Could not resolve host: github.com" + ) + return super().bridge_to_foundation( + vault, + foundation, + input_fn=input_fn, + output_fn=output_fn, + ) + + runtime = BlippedBridgeRuntime(_identity("1.81.0", "b")) + run = _execute( + tmp_path, + runtime, + source_repo=source_repo, + source_commit=source_commit, + ) + + assert run.case["reached_foundation"] is True + assert runtime.calls.count("bridge") == 2 + transcript = json.loads( + (tmp_path / "case.evidence" / "journey-transcript.json").read_text() + ) + events = {event["id"]: event for event in transcript["events"]} + assert events["bridge-foundation"]["journey_transport"] == {"attempt_count": 2} + assert events["bridge-foundation"]["approval_count"] == 3 + assert len(events["bridge-foundation"]["approvals"]) == 3 + + +def test_non_network_bridge_failure_is_never_retried( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_repo, source_commit = _executor_source_commit(tmp_path) + monkeypatch.setattr( + executor.time, + "sleep", + lambda _delay: pytest.fail("a non-network bridge failure must not back off"), + ) + + class RefusingBridgeRuntime(_Runtime): + def bridge_to_foundation(self, vault: Path, foundation, *, input_fn, output_fn): + self.calls.append("bridge") + raise executor.ExecutorError( + "foundation cache does not match the pinned official release" + ) + + runtime = RefusingBridgeRuntime(_identity("1.81.0", "b")) + with pytest.raises(executor.ExecutorError, match="foundation cache"): + _execute( + tmp_path, + runtime, + source_repo=source_repo, + source_commit=source_commit, + ) + + assert runtime.calls.count("bridge") == 1 + + def test_controlled_fleet_approval_still_rejects_excess_bridge_prompts( tmp_path: Path, ) -> None: diff --git a/core/tests/test_release_fleet_failure_diagnostics.py b/core/tests/test_release_fleet_failure_diagnostics.py index eaae3154..24f8eb68 100644 --- a/core/tests/test_release_fleet_failure_diagnostics.py +++ b/core/tests/test_release_fleet_failure_diagnostics.py @@ -278,6 +278,7 @@ def test_failed_journey_retains_only_private_sanitized_diagnostic( } else: assert document["delivery"] == { + "attempt_count": 1, "elapsed_ms": document["delivery"]["elapsed_ms"], "failure_reason": "unclassified", "release_matches_expected": False, @@ -301,6 +302,7 @@ def test_closed_delivery_mismatch_is_shared_public_route_drift_with_safe_identit document = json.loads(diagnostic.read_text(encoding="utf-8")) assert document["delivery"] == { + "attempt_count": 1, "delivered_release": _identity("1.81.6", "c"), "elapsed_ms": document["delivery"]["elapsed_ms"], "failure_reason": "public-route-drift", @@ -360,6 +362,7 @@ def test_failure_diagnostic_retains_only_allowlisted_delivery_reason( ) assert document["delivery"] == { + "attempt_count": 1, "elapsed_ms": 900_000, "failure_reason": "evidence-invalid", "release_matches_expected": False, diff --git a/core/update/journey-protocol-v1.json b/core/update/journey-protocol-v1.json index 092e3e5d..7b1768a8 100644 --- a/core/update/journey-protocol-v1.json +++ b/core/update/journey-protocol-v1.json @@ -13,7 +13,7 @@ "follow-up-smoke" ], "executor": { - "sha256": "d7469d68ec133b4872a8eab0308078d45ba97101ae15300fcd2cc8f57c877366", + "sha256": "d84f6d0759b573e49ec7adefc4e2944201d767d5a52416c990647109b97c4c66", "source_path": "scripts/release_fleet_executor.py" }, "foundation_to_follow_up": { diff --git a/scripts/release_fleet_executor.py b/scripts/release_fleet_executor.py index 790d9a5d..40b4a249 100644 --- a/scripts/release_fleet_executor.py +++ b/scripts/release_fleet_executor.py @@ -71,6 +71,50 @@ } ) _MAX_FAILURE_DIAGNOSTIC_ELAPSED_MS = 900_000 +# Harness-level protection for momentary controller network loss. The +# installed foundation's own updater predates in-release retry fixes, so the +# executor retries only transient network failures around each delivery hop. +_TRANSIENT_DELIVERY_MAX_ATTEMPTS = 3 +_TRANSIENT_DELIVERY_BACKOFF_SECONDS = (10.0, 30.0) +_TRANSIENT_NETWORK_DELIVERY_REASONS = frozenset({"network-unavailable"}) +_TRANSIENT_NETWORK_ERROR_FRAGMENTS = ( + "connection refused", + "connection reset", + "connection timed out", + "could not resolve host", + "couldn't connect to server", + "name or service not known", + "network is unreachable", + "network-unavailable", + "nodename nor servname provided", + "operation timed out", + "ssl connect error", + "temporary failure in name resolution", +) + + +def _transient_network_error(detail: str) -> bool: + """Classify one delivery error message as a momentary network outage.""" + + lowered = detail.lower() + return any( + fragment in lowered for fragment in _TRANSIENT_NETWORK_ERROR_FRAGMENTS + ) + + +def _transient_network_delivery(delivery: Mapping[str, object]) -> bool: + """Classify one not-delivered lifecycle response as transient network loss.""" + + evidence = delivery.get("evidence") + reason = evidence.get("reason") if isinstance(evidence, Mapping) else None + return isinstance(reason, str) and reason in _TRANSIENT_NETWORK_DELIVERY_REASONS + + +def _transient_delivery_backoff(attempt_number: int) -> None: + """Wait briefly before the next bounded transient-network attempt.""" + + delays = _TRANSIENT_DELIVERY_BACKOFF_SECONDS + time.sleep(delays[min(attempt_number, len(delays)) - 1]) class ExecutorError(RuntimeError): @@ -1359,6 +1403,7 @@ def _failure_diagnostic( doctor: Mapping[str, object] | None = None, delivery: Mapping[str, object] | None = None, delivery_elapsed_ms: int | None = None, + delivery_attempt_count: int = 1, ) -> dict[str, object]: document: dict[str, object] = { "acceptance": False, @@ -1391,6 +1436,10 @@ def _failure_diagnostic( max(delivery_elapsed_ms or 0, 0), _MAX_FAILURE_DIAGNOSTIC_ELAPSED_MS, ), + "attempt_count": min( + max(delivery_attempt_count, 1), + _TRANSIENT_DELIVERY_MAX_ATTEMPTS, + ), } if route_drift_release is not None: delivery_diagnostic["delivered_release"] = route_drift_release @@ -1592,14 +1641,28 @@ def bridge_input(prompt: str) -> str: raise ExecutorError("foundation bridge approval was not exact") return answer - bridge_response = dict( - _runtime.bridge_to_foundation( - vault, - foundation, - input_fn=bridge_input, - output_fn=output_fn, - ) - ) + bridge_attempt_count = 0 + while True: + bridge_attempt_count += 1 + bridge_approvals.clear() + try: + bridge_response = dict( + _runtime.bridge_to_foundation( + vault, + foundation, + input_fn=bridge_input, + output_fn=output_fn, + ) + ) + except ExecutorError as error: + if ( + bridge_attempt_count >= _TRANSIENT_DELIVERY_MAX_ATTEMPTS + or not _transient_network_error(str(error)) + ): + raise + _transient_delivery_backoff(bridge_attempt_count) + continue + break if len(bridge_approvals) > protocol.bridge.approval_count: raise ExecutorError("foundation bridge requested too many approvals") bridge_result = _validate_bridge_result( @@ -1630,28 +1693,48 @@ def bridge_input(prompt: str) -> str: if after_foundation != before: raise ExecutorError("user-owned content changed during foundation health proof") - delivery_started = time.monotonic_ns() - delivered = dict(_runtime.deliver_latest_release(vault)) - delivery_elapsed_ms = (time.monotonic_ns() - delivery_started) // 1_000_000 - delivered_release = _public_route_drift_identity(delivered, follow_up) - if delivered_release is not None: - _write_failure_diagnostic( - evidence_root, - "follow-up-delivery-failure.diagnostic.json", - _failure_diagnostic( - phase="follow-up-delivery", - vault=vault, - foundation=foundation, - follow_up=follow_up, - delivery=delivered, - delivery_elapsed_ms=delivery_elapsed_ms, - ), - ) - raise PublicRouteDriftError(follow_up, delivered_release) - if ( - delivered.get("status") != "delivered" - or delivered.get("release") != follow_up - ): + delivery_attempt_count = 0 + while True: + delivery_attempt_count += 1 + delivery_started = time.monotonic_ns() + try: + delivered = dict(_runtime.deliver_latest_release(vault)) + except ExecutorError as error: + if ( + delivery_attempt_count >= _TRANSIENT_DELIVERY_MAX_ATTEMPTS + or not _transient_network_error(str(error)) + ): + raise + _transient_delivery_backoff(delivery_attempt_count) + continue + delivery_elapsed_ms = (time.monotonic_ns() - delivery_started) // 1_000_000 + delivered_release = _public_route_drift_identity(delivered, follow_up) + if delivered_release is not None: + _write_failure_diagnostic( + evidence_root, + "follow-up-delivery-failure.diagnostic.json", + _failure_diagnostic( + phase="follow-up-delivery", + vault=vault, + foundation=foundation, + follow_up=follow_up, + delivery=delivered, + delivery_elapsed_ms=delivery_elapsed_ms, + delivery_attempt_count=delivery_attempt_count, + ), + ) + raise PublicRouteDriftError(follow_up, delivered_release) + if ( + delivered.get("status") == "delivered" + and delivered.get("release") == follow_up + ): + break + if ( + delivery_attempt_count < _TRANSIENT_DELIVERY_MAX_ATTEMPTS + and _transient_network_delivery(delivered) + ): + _transient_delivery_backoff(delivery_attempt_count) + continue _write_failure_diagnostic( evidence_root, "follow-up-delivery-failure.diagnostic.json", @@ -1662,6 +1745,7 @@ def bridge_input(prompt: str) -> str: follow_up=follow_up, delivery=delivered, delivery_elapsed_ms=delivery_elapsed_ms, + delivery_attempt_count=delivery_attempt_count, ), ) raise ExecutorError("lifecycle delivery did not prove the requested follow-up release") @@ -1769,6 +1853,7 @@ def bridge_input(prompt: str) -> str: "bridge_asset": dict(bridge_asset_evidence), "approval_count": len(bridge_approvals), "approvals": bridge_approvals, + "journey_transport": {"attempt_count": bridge_attempt_count}, }, { "id": "foundation-update-surface", @@ -1783,6 +1868,7 @@ def bridge_input(prompt: str) -> str: "from_release": foundation, "target_release": follow_up, "preview_sha256": preview_sha256, + "journey_transport": {"attempt_count": delivery_attempt_count}, }, { "id": "foundation-approval",