diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b58edaa..50920d8 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -135,7 +135,7 @@ jobs: key: podman-images-v2-integration-${{ steps.digests.outputs.hash }} - name: Run integration tests - run: sudo make test-integration TEST_PROCS=3 + run: sudo make test-integration TEST_PROCS=2 timeout-minutes: 90 env: CONTAINER_HOST: unix:///run/podman/podman.sock diff --git a/internal/cli/cluster/start.go b/internal/cli/cluster/start.go index af08df1..8c81f7e 100644 --- a/internal/cli/cluster/start.go +++ b/internal/cli/cluster/start.go @@ -183,6 +183,9 @@ func runStart(ctx context.Context, logger *logrus.Logger, nodeName string, nodeI if err := haproxyMgr.EnsureHAProxy(ctx, 0); err != nil { return fmt.Errorf("creating HAProxy load balancer: %w", err) } + if err := haproxyMgr.WaitForHealthy(ctx); err != nil { + return fmt.Errorf("waiting for HAProxy: %w", err) + } logger.Info("") logger.Info("✅ Cluster created successfully!") diff --git a/internal/cluster/join.go b/internal/cluster/join.go index fc11b18..2c31601 100644 --- a/internal/cluster/join.go +++ b/internal/cluster/join.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/bootc-dev/bink/internal/config" "github.com/bootc-dev/bink/internal/ssh" ) @@ -101,17 +102,24 @@ func (c *Cluster) Join(ctx context.Context, opts JoinOptions) error { c.logger.Info("") c.logger.Infof("=== Labeling %s ===", nodeName) - containerName := fmt.Sprintf("k8s-%s-%s", c.name, controlPlane) - kubeClient, err := c.newKubeClient(ctx, cpSSHClient, containerName) + haproxyContainer := fmt.Sprintf("%s%s-%s", config.ContainerNamePrefix, c.name, config.HAProxyContainerName) + kubeClient, err := c.newKubeClient(ctx, cpSSHClient, haproxyContainer) if err != nil { - c.logger.Warnf("Failed to create kubernetes client (non-fatal): %v", err) - } else { - if err := kubeClient.LabelNode(ctx, nodeName, labels); err != nil { - c.logger.Warnf("Failed to label node (non-fatal): %v", err) - } else { - c.logger.Infof("✅ Node %s labeled", nodeName) + return fmt.Errorf("labeling node %s: creating kubernetes client: %w", nodeName, err) + } + var labelErr error + for attempt := 1; attempt <= 5; attempt++ { + labelErr = kubeClient.LabelNode(ctx, nodeName, labels) + if labelErr == nil { + break } + c.logger.Warnf("Failed to label node (attempt %d/5): %v", attempt, labelErr) + time.Sleep(5 * time.Second) + } + if labelErr != nil { + return fmt.Errorf("labeling node %s: %w", nodeName, labelErr) } + c.logger.Infof("✅ Node %s labeled", nodeName) } c.logger.Info("") diff --git a/internal/haproxy/haproxy.go b/internal/haproxy/haproxy.go index 14221dc..f5c128f 100644 --- a/internal/haproxy/haproxy.go +++ b/internal/haproxy/haproxy.go @@ -5,9 +5,12 @@ package haproxy import ( "context" + "crypto/tls" "fmt" + "net/http" "strings" "text/template" + "time" "github.com/bootc-dev/bink/internal/config" "github.com/bootc-dev/bink/internal/podman" @@ -202,6 +205,56 @@ func (m *Manager) GetPublishedPort(ctx context.Context) (int, error) { return m.podman.GetPublishedPort(ctx, m.containerName(), fmt.Sprintf("%d/tcp", config.HAProxyPort)) } +// WaitForHealthy polls the Kubernetes API /healthz endpoint through HAProxy +// until it returns HTTP 200, indicating that HAProxy has healthy backends. +func (m *Manager) WaitForHealthy(ctx context.Context) error { + port, err := m.GetPublishedPort(ctx) + if err != nil { + return fmt.Errorf("getting HAProxy published port: %w", err) + } + + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + + healthURL := fmt.Sprintf("https://localhost:%d/healthz", port) + timer := time.NewTimer(2 * time.Minute) + defer timer.Stop() + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + logrus.Info("Waiting for HAProxy to become healthy...") + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return fmt.Errorf("timed out waiting for HAProxy to become healthy on port %d", port) + case <-ticker.C: + reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, healthURL, nil) + if err != nil { + cancel() + continue + } + resp, err := client.Do(req) + if err != nil { + cancel() + continue + } + resp.Body.Close() + cancel() + if resp.StatusCode == http.StatusOK { + logrus.Info("HAProxy is healthy") + return nil + } + } + } +} + // discoverBackends finds all control-plane node containers in this cluster // and returns their bridge IPs as backends. func (m *Manager) discoverBackends(ctx context.Context) ([]backend, error) {