From 32b2eb42f7000850b9bffcdef59daa88c8d95916 Mon Sep 17 00:00:00 2001 From: gamnaansong Date: Sat, 13 Jun 2026 08:55:14 +0000 Subject: [PATCH] fix(runner): drop dead-on-arrival toolbox HTTP readiness check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit waitForToolboxReady was introduced in fc88aa0b (2026-06-05, "feat: add agent-ready runtime catalog", which then landed on main via PR #715 on 2026-06-10). It HTTP-polls http://127.0.0.1:/version expecting a daemon on guest TCP port 2280 — that interface dates from the Daytona-daemon era and was never reimplemented after the Rust guest agent rewrite landed in dbb11ec8 (2026-04-01). The new agent binds **vsock://2695** for gRPC + notifies the host via vsock://2696; nothing inside the VM listens on TCP:2280, so libkrun's port-forward accepts the SYN and immediately reset-by-peer's, and every CREATE_BOX times out 30 s in. Production data from a Tokyo runner: in 24 h, 490 CREATE_BOX events, 0 toolbox-ready successes, 181 toolbox-ready failures. The exec path that fires immediately afterward (via the same vsock gRPC channel) **always succeeds** — confirming the box VM is healthy, the readiness check itself is the bug. Remove the dead probe: drop waitForToolboxReady from client.go's Create and Start, drop the function + its TCP/HTTP imports, drop the toolboxReadyTimeout field, drop the ToolboxReadyTimeout/ DaemonStartTimeoutSec config plumbing in main.go + config.go, drop the two now-unreachable tests. Box readiness is now signalled by bx.Start(ctx) returning (which itself blocks on the vsock notification from the guest). Branched off chore/e2e-required-merge-gate (PR #724) so the e2e-cloud stack picks this up next dispatch. --- apps/runner/cmd/runner/config/config.go | 1 - apps/runner/cmd/runner/main.go | 1 - apps/runner/pkg/boxlite/client.go | 14 ----- apps/runner/pkg/boxlite/toolbox_ports.go | 52 ---------------- apps/runner/pkg/boxlite/toolbox_ports_test.go | 60 ------------------- 5 files changed, 128 deletions(-) diff --git a/apps/runner/cmd/runner/config/config.go b/apps/runner/cmd/runner/config/config.go index f5880a03b..0d213c467 100644 --- a/apps/runner/cmd/runner/config/config.go +++ b/apps/runner/cmd/runner/config/config.go @@ -37,7 +37,6 @@ type Config struct { AWSSecretAccessKey string `envconfig:"AWS_SECRET_ACCESS_KEY"` AWSDefaultBucket string `envconfig:"AWS_DEFAULT_BUCKET"` ResourceLimitsDisabled bool `envconfig:"RESOURCE_LIMITS_DISABLED"` - DaemonStartTimeoutSec int `envconfig:"DAEMON_START_TIMEOUT_SEC"` BoxStartTimeoutSec int `envconfig:"BOX_START_TIMEOUT_SEC"` UseSnapshotEntrypoint bool `envconfig:"USE_SNAPSHOT_ENTRYPOINT"` Domain string `envconfig:"RUNNER_DOMAIN" validate:"omitempty,hostname|ip"` diff --git a/apps/runner/cmd/runner/main.go b/apps/runner/cmd/runner/main.go index 1785114c1..4d3ea0f0f 100644 --- a/apps/runner/cmd/runner/main.go +++ b/apps/runner/cmd/runner/main.go @@ -118,7 +118,6 @@ func run() int { VolumeCleanupInterval: cfg.VolumeCleanupInterval, VolumeCleanupDryRun: cfg.VolumeCleanupDryRun, VolumeCleanupExclusionPeriod: cfg.VolumeCleanupExclusionPeriod, - ToolboxReadyTimeout: time.Duration(cfg.DaemonStartTimeoutSec) * time.Second, }) if err != nil { logger.Error("Error creating BoxLite client", "error", err) diff --git a/apps/runner/pkg/boxlite/client.go b/apps/runner/pkg/boxlite/client.go index 537b3f9fe..4ac45ad5a 100644 --- a/apps/runner/pkg/boxlite/client.go +++ b/apps/runner/pkg/boxlite/client.go @@ -37,7 +37,6 @@ type Client struct { volumeMutexesMutex sync.Mutex volumeCleanupMutex sync.Mutex toolboxPortMutex sync.Mutex - toolboxReadyTimeout time.Duration lastVolumeCleanup time.Time volumeCleanup volumeCleanupConfig } @@ -56,7 +55,6 @@ type ClientConfig struct { VolumeCleanupInterval time.Duration VolumeCleanupDryRun bool VolumeCleanupExclusionPeriod time.Duration - ToolboxReadyTimeout time.Duration } func networkSpec(blockAll *bool, allowList *string) boxlite.NetworkSpec { @@ -136,11 +134,6 @@ func buildImageRegistries(insecureRegistries []string, ghcrUsername, ghcrToken s // NewClient creates a new BoxLite client backed by the BoxLite VM runtime. func NewClient(ctx context.Context, config ClientConfig) (*Client, error) { - toolboxReadyTimeout := config.ToolboxReadyTimeout - if toolboxReadyTimeout <= 0 { - toolboxReadyTimeout = 30 * time.Second - } - var opts []boxlite.RuntimeOption if config.HomeDir != "" { opts = append(opts, boxlite.WithHomeDir(config.HomeDir)) @@ -171,7 +164,6 @@ func NewClient(ctx context.Context, config ClientConfig) (*Client, error) { awsAccessKeyId: config.AWSAccessKeyId, awsSecretAccessKey: config.AWSSecretAccessKey, volumeMutexes: make(map[string]*sync.Mutex), - toolboxReadyTimeout: toolboxReadyTimeout, volumeCleanup: volumeCleanupConfig{ interval: config.VolumeCleanupInterval, dryRun: config.VolumeCleanupDryRun, @@ -305,9 +297,6 @@ func (c *Client) Create(ctx context.Context, boxDto dto.CreateBoxDTO) (string, s if err := bx.Start(ctx); err != nil { return bx.ID(), "", fmt.Errorf("failed to start box: %w", err) } - if err := c.waitForToolboxReady(ctx, boxDto.Id); err != nil { - return bx.ID(), "", err - } } return bx.ID(), "boxlite", nil @@ -326,9 +315,6 @@ func (c *Client) Start(ctx context.Context, boxId string, authToken *string, met if err := bx.Start(ctx); err != nil { return "", err } - if err := c.waitForToolboxReady(ctx, boxId); err != nil { - return "", err - } return "boxlite", nil } diff --git a/apps/runner/pkg/boxlite/toolbox_ports.go b/apps/runner/pkg/boxlite/toolbox_ports.go index f9634bddb..6c830ae10 100644 --- a/apps/runner/pkg/boxlite/toolbox_ports.go +++ b/apps/runner/pkg/boxlite/toolbox_ports.go @@ -7,14 +7,11 @@ import ( "context" "encoding/json" "fmt" - "io" "net" - "net/http" "os" "path/filepath" "strconv" "strings" - "time" ) const ( @@ -62,55 +59,6 @@ func (c *Client) ToolboxHostPort(boxID string) (int, error) { return c.readToolboxHostPort(boxID) } -func (c *Client) waitForToolboxReady(ctx context.Context, boxID string) error { - hostPort, err := c.ToolboxHostPort(boxID) - if err != nil { - return fmt.Errorf("toolbox host port not available for box %s: %w", boxID, err) - } - - timeout := c.toolboxReadyTimeout - if timeout <= 0 { - timeout = 30 * time.Second - } - readyCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - url := fmt.Sprintf("http://127.0.0.1:%d/version", hostPort) - client := http.Client{Timeout: time.Second} - ticker := time.NewTicker(200 * time.Millisecond) - defer ticker.Stop() - - var lastErr error - for { - req, reqErr := http.NewRequestWithContext(readyCtx, http.MethodGet, url, nil) - if reqErr != nil { - return reqErr - } - - resp, reqErr := client.Do(req) - if reqErr == nil { - _, _ = io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() - if resp.StatusCode >= 200 && resp.StatusCode < 300 { - c.logger.InfoContext(ctx, "box toolbox is ready", "box", boxID, "hostPort", hostPort) - return nil - } - lastErr = fmt.Errorf("unexpected status %d from %s", resp.StatusCode, url) - } else { - lastErr = reqErr - } - - select { - case <-readyCtx.Done(): - if lastErr != nil { - return fmt.Errorf("box toolbox not ready after %s (box=%s hostPort=%d): %w", timeout, boxID, hostPort, lastErr) - } - return fmt.Errorf("box toolbox not ready after %s (box=%s hostPort=%d)", timeout, boxID, hostPort) - case <-ticker.C: - } - } -} - func (c *Client) removeToolboxPortRecord(ctx context.Context, boxID string) error { c.toolboxPortMutex.Lock() defer c.toolboxPortMutex.Unlock() diff --git a/apps/runner/pkg/boxlite/toolbox_ports_test.go b/apps/runner/pkg/boxlite/toolbox_ports_test.go index 8db4f9ccf..2861beeda 100644 --- a/apps/runner/pkg/boxlite/toolbox_ports_test.go +++ b/apps/runner/pkg/boxlite/toolbox_ports_test.go @@ -6,10 +6,7 @@ package boxlite import ( "io" "log/slog" - "net" - "net/http" "testing" - "time" ) func TestReserveToolboxHostPortPersistsRecord(t *testing.T) { @@ -55,60 +52,3 @@ func TestRemoveToolboxPortRecord(t *testing.T) { } } -func TestWaitForToolboxReadyReturnsAfterVersionEndpointResponds(t *testing.T) { - listener, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatalf("listen: %v", err) - } - mux := http.NewServeMux() - mux.HandleFunc("/version", func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`{"version":"test"}`)) - }) - server := &http.Server{Handler: mux} - go func() { - _ = server.Serve(listener) - }() - defer server.Close() - - port := listener.Addr().(*net.TCPAddr).Port - client := &Client{ - homeDir: t.TempDir(), - logger: slog.New(slog.NewTextHandler(io.Discard, nil)), - toolboxReadyTimeout: time.Second, - } - if err := client.writeToolboxPortRecord(toolboxPortRecord{ - BoxID: "box-1", - GuestPort: ToolboxGuestPort, - HostPort: port, - }); err != nil { - t.Fatalf("writeToolboxPortRecord: %v", err) - } - - if err := client.waitForToolboxReady(t.Context(), "box-1"); err != nil { - t.Fatalf("waitForToolboxReady: %v", err) - } -} - -func TestWaitForToolboxReadyTimesOutWhenEndpointDoesNotRespond(t *testing.T) { - port, err := findAvailableLocalPort() - if err != nil { - t.Fatalf("findAvailableLocalPort: %v", err) - } - client := &Client{ - homeDir: t.TempDir(), - logger: slog.New(slog.NewTextHandler(io.Discard, nil)), - toolboxReadyTimeout: 20 * time.Millisecond, - } - if err := client.writeToolboxPortRecord(toolboxPortRecord{ - BoxID: "box-1", - GuestPort: ToolboxGuestPort, - HostPort: port, - }); err != nil { - t.Fatalf("writeToolboxPortRecord: %v", err) - } - - if err := client.waitForToolboxReady(t.Context(), "box-1"); err == nil { - t.Fatal("expected waitForToolboxReady to time out") - } -}