-
Notifications
You must be signed in to change notification settings - Fork 66
fix(attach): isolate registration stalls and back off on database overload #1706
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| { | ||
| "id": "compact_msichwe8jqzl", | ||
| "version": 1, | ||
| "type": "compacted", | ||
| "compactedAt": "2026-09-08T10:22:43.328Z", | ||
| "sourceTrajectories": [ | ||
| "traj_pg9aglu4aqlw" | ||
| ], | ||
| "dateRange": { | ||
| "start": "2026-09-08T10:04:45.586Z", | ||
| "end": "2026-09-08T10:21:50.230Z" | ||
| }, | ||
| "summary": { | ||
| "totalDecisions": 1, | ||
| "totalEvents": 1, | ||
| "uniqueAgents": [ | ||
| "default" | ||
| ] | ||
| }, | ||
| "decisionGroups": [ | ||
| { | ||
| "category": "api", | ||
| "decisions": [ | ||
| { | ||
| "question": "Move HTTP-spawn cloud preparation out of the broker actor and bound it", | ||
| "chosen": "Move HTTP-spawn cloud preparation out of the broker actor and bound it", | ||
| "reasoning": "Wrangler traced a 35.7s duplicate registration blocking local attach. Preserve node tokens, merge only completed identity/cursor state, reserve local names, reject pending releases, and skip launch when the caller is gone. Retry only explicit pre-allocation D1 overload for fleet attach.", | ||
| "fromTrajectory": "traj_pg9aglu4aqlw" | ||
| } | ||
| ] | ||
| } | ||
| ], | ||
| "keyLearnings": [], | ||
| "keyFindings": [], | ||
| "filesAffected": [], | ||
| "commits": [] | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| # Trajectory Compaction: Sep 8, 2026 - Sep 8, 2026 | ||
|
|
||
| ## Summary | ||
| - Sessions: 1 | ||
| - Decisions: 1 | ||
| - Events: 1 | ||
| - Agents: default | ||
| - Files: 0 | ||
| - Commits: 0 | ||
|
|
||
| ## Api | ||
| - Move HTTP-spawn cloud preparation out of the broker actor and bound it -> Move HTTP-spawn cloud preparation out of the broker actor and bound it (traj_pg9aglu4aqlw) | ||
|
|
||
| ## Key Learnings | ||
| - None | ||
|
|
||
| ## Key Findings | ||
| - None |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| { | ||
| "trajectoryId": "traj_pg9aglu4aqlw", | ||
| "compactedInto": "compact_msichwe8jqzl", | ||
| "compactedAt": "2026-09-08T10:22:43.428Z" | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| # Trajectory: Keep local attach responsive during slow cloud registration | ||
|
|
||
| > **Status:** ✅ Completed | ||
| > **Confidence:** 90% | ||
| > **Started:** September 8, 2026 at 12:04 PM | ||
| > **Completed:** September 8, 2026 at 12:21 PM | ||
|
|
||
| --- | ||
|
|
||
| ## Summary | ||
|
|
||
| Isolated HTTP-spawn cloud registration from local attach; bounded registration and overload retries; accurate health/attach diagnostics. 1065 broker tests passed (4 ignored), 43 attach tests passed, CLI typecheck passed. Same real-pipe regression fails on baseline 11.10.4 and passes with fix. Production broker untouched; rollout pending. | ||
|
|
||
| **Approach:** Standard approach | ||
|
|
||
| --- | ||
|
|
||
| ## Key Decisions | ||
|
|
||
| ### Move HTTP-spawn cloud preparation out of the broker actor and bound it | ||
| - **Chose:** Move HTTP-spawn cloud preparation out of the broker actor and bound it | ||
| - **Reasoning:** Wrangler traced a 35.7s duplicate registration blocking local attach. Preserve node tokens, merge only completed identity/cursor state, reserve local names, reject pending releases, and skip launch when the caller is gone. Retry only explicit pre-allocation D1 overload for fleet attach. | ||
|
|
||
| --- | ||
|
|
||
| ## Chapters | ||
|
|
||
| ### 1. Work | ||
| *Agent: default* | ||
|
|
||
| - Move HTTP-spawn cloud preparation out of the broker actor and bound it: Move HTTP-spawn cloud preparation out of the broker actor and bound it |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| { | ||
| "id": "traj_pg9aglu4aqlw", | ||
| "version": 1, | ||
| "task": { | ||
| "title": "Keep local attach responsive during slow cloud registration" | ||
| }, | ||
| "status": "completed", | ||
| "startedAt": "2026-09-08T10:04:45.586Z", | ||
| "completedAt": "2026-09-08T10:21:50.230Z", | ||
| "agents": [ | ||
| { | ||
| "name": "default", | ||
| "role": "lead", | ||
| "joinedAt": "2026-09-08T10:16:44.611Z" | ||
| } | ||
| ], | ||
| "chapters": [ | ||
| { | ||
| "id": "chap_k90zf0m041ko", | ||
| "title": "Work", | ||
| "agentName": "default", | ||
| "startedAt": "2026-09-08T10:16:44.611Z", | ||
| "endedAt": "2026-09-08T10:21:50.230Z", | ||
| "events": [ | ||
| { | ||
| "ts": 1788862604611, | ||
| "type": "decision", | ||
| "content": "Move HTTP-spawn cloud preparation out of the broker actor and bound it: Move HTTP-spawn cloud preparation out of the broker actor and bound it", | ||
| "raw": { | ||
| "question": "Move HTTP-spawn cloud preparation out of the broker actor and bound it", | ||
| "chosen": "Move HTTP-spawn cloud preparation out of the broker actor and bound it", | ||
| "alternatives": [], | ||
| "reasoning": "Wrangler traced a 35.7s duplicate registration blocking local attach. Preserve node tokens, merge only completed identity/cursor state, reserve local names, reject pending releases, and skip launch when the caller is gone. Retry only explicit pre-allocation D1 overload for fleet attach." | ||
| }, | ||
| "significance": "high" | ||
| } | ||
| ] | ||
| } | ||
| ], | ||
| "retrospective": { | ||
| "summary": "Isolated HTTP-spawn cloud registration from local attach; bounded registration and overload retries; accurate health/attach diagnostics. 1065 broker tests passed (4 ignored), 43 attach tests passed, CLI typecheck passed. Same real-pipe regression fails on baseline 11.10.4 and passes with fix. Production broker untouched; rollout pending.", | ||
| "approach": "Standard approach", | ||
| "confidence": 0.9 | ||
| }, | ||
| "commits": [], | ||
| "filesChanged": [], | ||
| "projectId": "AgentWorkforce/relay", | ||
| "tags": [], | ||
| "_trace": { | ||
| "startRef": "d754fff143c464c367ac743ccc3c8085bf5ef04d", | ||
| "endRef": "d754fff143c464c367ac743ccc3c8085bf5ef04d" | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change | ||||||
|---|---|---|---|---|---|---|---|---|
|
|
@@ -5,7 +5,13 @@ All notable changes to Agent Relay will be documented in this file. | |||||||
| The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), | ||||||||
| and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). | ||||||||
|
|
||||||||
| ## [Unreleased] | ||||||||
| ## [Unreleased - Patch] | ||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 📐 Maintainability & Code Quality | 🟡 Minor | ⚡ Quick win Use the standard The repository guideline requires the root As per coding guidelines, 🤖 Prompt for AI AgentsSource: Coding guidelines |
||||||||
|
|
||||||||
| ### Fixed | ||||||||
|
|
||||||||
| - Local agent attach, snapshots, and input remain responsive while HTTP spawns wait for cloud registration; registration now has bounded concurrency and an overall deadline. | ||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P3: This bullet ends with implementation backstory ('registration now has bounded concurrency and an overall deadline'), which the changelog convention says to drop. Keep only the user-visible impact and name the concrete command surface (e.g. Prompt for AI agents
Suggested change
|
||||||||
| - Fleet attach retries explicit database-overload rejections with server-directed backoff, and attach lookup failures explain that the agent may still be running. | ||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P3: This bullet combines two separate behaviors: retrying database-overload rejections, and explaining attach lookup failures. Split them into one bullet each, and for the retry one name the concrete attempt bound instead of generic 'retries ... with backoff' wording, per the repo changelog convention. Prompt for AI agents
Suggested change
|
||||||||
| - Broker health marks runtime status as unavailable instead of claiming zero agents when its status lookup times out. | ||||||||
|
|
||||||||
| ## [11.10.4] - 2026-09-08 | ||||||||
|
|
||||||||
|
|
||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -612,6 +612,29 @@ async fn listen_api_health( | |
| let mut payload = listen_api_health_payload(state.default_workspace_id, state.memberships); | ||
| if let Some(status) = fetch_status_for_health(&state.tx).await { | ||
| merge_status_into_health_payload(&mut payload, &status); | ||
| payload["runtimeResponsive"] = json!(true); | ||
| } else { | ||
| // Keep the existing public liveness HTTP contract: a busy actor must | ||
| // not induce a supervisor restart storm. But never present fabricated | ||
| // zero-agent/offline values as a successful runtime observation. | ||
| payload["runtimeResponsive"] = json!(false); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: While the broker is still waiting for Relaycast readiness, Prompt for AI agents |
||
| if payload["status"] == "ok" { | ||
| payload["status"] = json!("degraded"); | ||
| } | ||
| payload["runtimeStatusError"] = json!("runtime_status_unavailable"); | ||
| for key in [ | ||
| "agentCount", | ||
| "uptimeMs", | ||
| "pendingDeliveryCount", | ||
| "deadLetterCount", | ||
| "wsConnections", | ||
| "memoryMb", | ||
| "relaycastConnected", | ||
| "nodeConnected", | ||
| "nodeDelivery", | ||
| ] { | ||
| payload[key] = Value::Null; | ||
| } | ||
| } | ||
| axum::Json(payload) | ||
| } | ||
|
|
@@ -3880,6 +3903,35 @@ mod auth_tests { | |
| .expect("request should succeed"); | ||
|
|
||
| assert_eq!(response.status(), StatusCode::OK); | ||
| let payload = response_json(response).await; | ||
| assert_eq!(payload["runtimeResponsive"], false); | ||
| assert_eq!(payload["runtimeStatusError"], "runtime_status_unavailable"); | ||
| assert!(payload["agentCount"].is_null()); | ||
| assert!(payload["nodeConnected"].is_null()); | ||
| } | ||
|
|
||
| #[tokio::test] | ||
| async fn health_route_reports_a_responsive_runtime_without_losing_counts() { | ||
| let (router, mut rx) = test_router(None); | ||
| let runtime = tokio::spawn(async move { | ||
| if let Some(super::ListenApiRequest::GetStatus { reply }) = rx.recv().await { | ||
| let _ = reply.send(Ok(json!({ "agent_count": 1 }))); | ||
| } | ||
| }); | ||
| let response = router | ||
| .oneshot( | ||
| Request::builder() | ||
| .uri("/health") | ||
| .body(Body::empty()) | ||
| .unwrap(), | ||
| ) | ||
| .await | ||
| .unwrap(); | ||
| assert_eq!(response.status(), StatusCode::OK); | ||
| let payload = response_json(response).await; | ||
| assert_eq!(payload["runtimeResponsive"], true); | ||
| assert_eq!(payload["agentCount"], 1); | ||
| runtime.await.unwrap(); | ||
| } | ||
|
|
||
| #[test] | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P3: This completed trajectory records no commits and no changed files, and its
_trace.startRefequalsendRef, so it does not span the work it claims to represent. PopulatecommitsandfilesChangedwith the actual PR commits/files and setstartRef/endRefto distinct refs covering them so the audit trail is accurate. Since this is generated by the Trail/CLI tooling, regenerate from the owning renderer rather than hand-editing if that source is available.Prompt for AI agents