fix(operator): pace the automatic retries of an interrupted apply, and skip them when the target's verdict is final

TEMPLATES.md

@@ -3259,7 +3259,7 @@ _Last updated: <relative-time datetime="2026-01-01T00:00:00Z">2026-01-01 00:00:0
 
 **Schema `testapp`**
 
-**`users`**: 🟧🟧🟧🟧🟧🟧⬜⬜⬜⬜⬜⬜⬜⬜⬜⬜⬜⬜⬜⬜ 🔄 Interrupted — retrying automatically (attempt 2/10)
+**`users`**: 🟧🟧🟧🟧🟧🟧⬜⬜⬜⬜⬜⬜⬜⬜⬜⬜⬜⬜⬜⬜ 🔄 Retrying · attempt 4/10 · next 14:31 UTC
 
 ```sql
 ALTER TABLE `users` ADD INDEX `idx_email`(`email`);
@@ -4741,6 +4741,31 @@ Single table progress (default):
 
 ✓ Apply complete!
 
+```
+</details>
+
+<details>
+<summary><a name="mysql-single-table-retrying"></a><strong>MySQL: Single Table Retrying</strong></summary>
+
+```
+
+┌────────────────────────────────────────────────┐
+│  Apply ID:  apply-a1b2c3d4e5f6                 │
+│  State:     Retrying                           │
+│  Retry:     attempt 4/10 · next 14:31:00 UTC   │
+│  Started:   Jan 15 14:26:00 UTC                │
+│  Duration:  4m                                 │
+└────────────────────────────────────────────────┘
+
+  connection reset by peer
+
+
+  ── testapp ──
+
+     ~ users: 🟧🟧🟧🟧🟧🟧🟧⬜⬜⬜⬜⬜⬜⬜⬜⬜⬜⬜⬜⬜ Retrying
+       ALTER TABLE `users` ADD INDEX `idx_email_created`(`email`, `created_at`);
+
+
 ```
 </details>
 

docs/apply-lifecycle.md

@@ -159,9 +159,18 @@ data plane, a network blip, an unclean kill mid-drive. Nothing about the work
 needs to change; it just needs to be picked up again. Those failures move the
 apply to `failed_retryable` instead of `failed`, and recovery is automatic: a
 recovery driver reclaims the apply and continues it from where it left off,
-using engine checkpoints so completed work is not redone. Failures where
-retrying cannot help — the engine rejected a statement, the target refused the
-change — skip this state and go straight to permanent `failed`.
+using engine checkpoints so completed work is not redone.
+
+Failures where retrying cannot help skip this state and go straight to permanent
+`failed`. Those are the ones the target rejects for a reason that belongs to the
+statement or to the data already in the table: a duplicate value under a new
+unique index, existing rows that do not fit a narrowed column, a column the
+table does not have, DDL the target will not perform at all. Nothing about the
+target changes between attempts, so the apply is failed on the spot with the
+target's own reason rather than spending its whole budget reproducing it — and
+the database's active-apply slot is released for the corrected apply that
+follows. Anything that might read differently once a lock, a lagging replica, or
+a busy target has moved on keeps its retries.
 
 A clean shutdown is not one of these failures. A process that stops on purpose
 hands its claims back and leaves the apply active, so a peer driver resumes it
@@ -172,10 +181,24 @@ currently **10 attempts**, on top of the original run. The attempt counter is
 visible in the PR progress comment, so you can watch how much budget an apply
 has burned.
 
+Attempts are also **paced**. A failure that reproduces instantly — a refused
+lock, a connection error — would otherwise spend the whole budget in under a
+minute, on exactly the kind of failure a little time was most likely to clear.
+So each attempt arms a wait for the next one: the first couple of retries are
+immediate, and the wait then steps up and holds flat, spreading a fully spent
+budget over roughly ten minutes. The wait is measured from the *start* of an
+attempt, so an attempt that ran longer than its own wait has already spaced
+itself out and retries as soon as it fails. The PR comment and the CLI's
+progress view name the clock time the next attempt is due.
+
+The pacing applies to automatic retries only. An operator `start` runs now, and
+a driver that dies mid-attempt is picked up by a peer as soon as its lease goes
+stale — neither waits out a retry nobody asked for.
+
 ```
  running --(recoverable failure)--> failed_retryable
     ^                                     |
-    |     recovery driver reclaims,       |
+    |     recovery driver reclaims,       |  wait out the backoff
     +---- attempt counter +1, resume <----+
           from checkpoints                |
                                           |  budget spent (10 attempts),

pkg/api/ensure_schema_postgres_test.go

@@ -65,7 +65,7 @@ func TestPostgresCreateTableColumns_EmbeddedFiles(t *testing.T) {
 		"id", "apply_identifier", "lock_id", "plan_id", "database_name", "database_type",
 		"repository", "pull_request", "environment", "deployment", "caller", "installation_id",
 		"external_id", "idempotency_key", "engine", "state", "error_message", "options", "attempt",
-		"lease_owner", "lease_token", "lease_acquired_at", "started_at", "completed_at",
+		"retry_after", "lease_owner", "lease_token", "lease_acquired_at", "started_at", "completed_at",
 		"revert_skipped_at", "created_at", "updated_at",
 	}, applies)
 

pkg/api/handlers_test.go

@@ -6385,3 +6385,27 @@ func TestSetRevertSkippedMetadata(t *testing.T) {
 	setRevertSkippedMetadata(resp, &storage.Apply{RevertSkippedAt: &now})
 	assert.Equal(t, "true", resp.Metadata["revert_skipped"], "flag set once revert_skipped_at is present")
 }
+
+// overlayRetryBudget reports the spent retry budget in every state, so a
+// permanently failed apply still shows how many recoveries were tried, but names
+// the next attempt only while the apply is waiting for one. A claim leaves the
+// armed deadline behind on the row it resumes, so a resumed apply must not
+// advertise a retry that is not coming.
+func TestOverlayRetryBudget(t *testing.T) {
+	due := time.Now().Add(90 * time.Second)
+
+	retrying := &apitypes.ProgressResponse{}
+	overlayRetryBudget(retrying, &storage.Apply{State: state.Apply.FailedRetryable, Attempt: 3, RetryAfter: &due})
+	assert.Equal(t, int32(3), retrying.Attempt)
+	assert.Equal(t, due.Format(time.RFC3339), retrying.RetryAfter)
+
+	resumed := &apitypes.ProgressResponse{}
+	overlayRetryBudget(resumed, &storage.Apply{State: state.Apply.Running, Attempt: 3, RetryAfter: &due})
+	assert.Equal(t, int32(3), resumed.Attempt, "the spent budget stays visible after redispatch")
+	assert.Empty(t, resumed.RetryAfter, "a running apply is not waiting on a retry")
+
+	failed := &apitypes.ProgressResponse{}
+	overlayRetryBudget(failed, &storage.Apply{State: state.Apply.Failed, Attempt: storage.MaxRecoveryAttempts, RetryAfter: &due})
+	assert.Equal(t, int32(storage.MaxRecoveryAttempts), failed.Attempt, "an exhausted budget is the record of what was tried")
+	assert.Empty(t, failed.RetryAfter)
+}

pkg/api/progress_handlers.go

@@ -448,6 +448,8 @@ func (s *Service) handleProgressByApplyID(w http.ResponseWriter, r *http.Request
 
 	overlayApplyOptions(httpResp, apply)
 
+	overlayRetryBudget(httpResp, apply)
+
 	setRevertSkippedMetadata(httpResp, apply)
 
 	// Overlay per-table timestamps from task records. The proto response
@@ -474,6 +476,28 @@ func (s *Service) handleProgressByApplyID(w http.ResponseWriter, r *http.Request
 	s.writeJSON(w, http.StatusOK, httpResp)
 }
 
+// overlayRetryBudget surfaces the apply's automatic-retry state: how much of the
+// budget it has spent and, while it is waiting to be retried, when its next
+// attempt becomes eligible. Both are control-plane bookkeeping — the claim path
+// owns them — so they are read from the stored apply on every progress path,
+// including the one whose per-table detail comes from a remote engine.
+//
+// The spent budget is reported in every state: on a permanently failed apply it
+// is the record of how many recoveries were tried. The wait is reported only
+// while the apply is retrying, because the stored deadline outlives the state
+// that gave it meaning — a claim leaves the armed deadline behind on the row it
+// resumes, and reporting that on a running or completed apply would name a
+// retry that is not coming.
+func overlayRetryBudget(resp *apitypes.ProgressResponse, apply *storage.Apply) {
+	if apply == nil {
+		return
+	}
+	resp.Attempt = int32(apply.Attempt)
+	if apply.RetryAfter != nil && state.IsState(apply.State, state.Apply.FailedRetryable) {
+		resp.RetryAfter = apply.RetryAfter.Format(time.RFC3339)
+	}
+}
+
 // setRevertSkippedMetadata surfaces the skip-revert flag from the apply's stored
 // revert_skipped_at, so progress consumers can show that revert was skipped and
 // finalization is in progress. It reads apply state — no engine-specific side
@@ -1130,6 +1154,7 @@ func (s *Service) progressFromLocalStorage(ctx context.Context, apply *storage.A
 		httpResp.ErrorMessage = apply.ErrorMessage
 	}
 	overlayApplyOptions(httpResp, apply)
+	overlayRetryBudget(httpResp, apply)
 	setRevertSkippedMetadata(httpResp, apply)
 	operations, deploymentByOperationID, released := s.bestEffortProgressOperations(ctx, apply)
 	httpResp.Operations = operations

pkg/apitypes/apitypes.go

@@ -814,6 +814,16 @@ type ProgressResponse struct {
 	PullRequest string `json:"pull_request,omitempty"` // PR URL (blank for CLI context)
 	StartedAt   string `json:"started_at,omitempty"`
 	CompletedAt string `json:"completed_at,omitempty"`
+	// Attempt is how many automatic redispatches an interrupted apply has
+	// already consumed of its retry budget. Zero on an apply that has not been
+	// redispatched.
+	Attempt int32 `json:"attempt,omitempty"`
+	// RetryAfter is the RFC3339 time the next automatic retry becomes eligible.
+	// An interrupted apply backs off between attempts, so this distinguishes an
+	// apply that is waiting from one that is due and about to be picked up. Set
+	// only while the apply is retrying and a wait was armed; empty otherwise,
+	// including on an apply that has already been redispatched.
+	RetryAfter string `json:"retry_after,omitempty"`
 	// Operations carries per-deployment operation rows for multi-deployment applies.
 	// Empty for single-deployment applies.
 	Operations   []*ProgressOperationResponse `json:"operations,omitempty"`

pkg/cmd/commands/preview.go

@@ -35,8 +35,8 @@ func (cmd *PreviewCmd) Run(g *Globals) error {
 	switch previewType {
 	// Basic types
 	case templates.PreviewPlan, templates.PreviewProgress, templates.PreviewWaitingForDeploy, templates.PreviewWaitingForCutover,
-		templates.PreviewCuttingOver, templates.PreviewCompleted, templates.PreviewFailed,
-		templates.PreviewStopped, templates.PreviewStates:
+		templates.PreviewCuttingOver, templates.PreviewCompleted, templates.PreviewRetrying,
+		templates.PreviewFailed, templates.PreviewStopped, templates.PreviewStates:
 		templates.PreviewCLIOutput(previewType)
 	// Lock types
 	case templates.PreviewLockAcquired, templates.PreviewLockConflict,
@@ -173,6 +173,7 @@ Basic Types:
   waiting_for_cutover   Show sample waiting for cutover output
   cutting_over          Show sample cutting over output
   completed             Show sample completed output
+  retrying              Show sample interrupted output (waiting on an automatic retry)
   failed                Show sample failed output
   stopped               Show sample stopped output (mid-apply stop)
   states                Show state display formatting
@@ -276,7 +277,7 @@ Comment Templates (GitHub PR comments):
   comment_apply_completed       Multi-table: completed (all tables done)
   comment_apply_failed          Multi-table: failed (with error and cancelled tables)
   comment_apply_failed_before_row_copy Multi-table: failed before row copy (preflight rejection, per-table error)
-  comment_apply_retrying        Multi-table: interrupted, retrying automatically (attempt counter)
+  comment_apply_retrying        Multi-table: interrupted, retrying automatically (attempt counter + next attempt time)
   comment_apply_stopped         Multi-table: stopped (partial progress)
   comment_apply_waiting_cutover Waiting for cutover (deferred, operator triggers)
   comment_apply_waiting_cutover_automatic Waiting for cutover (non-deferred, drive triggers)

pkg/cmd/internal/templates/preview.go

@@ -13,6 +13,7 @@ var previewTime = time.Date(2026, 1, 15, 14, 30, 0, 0, time.UTC)
 // SetPreviewMode configures the package to use fixed timestamps for deterministic output.
 func SetPreviewMode() {
 	nowFunc = func() time.Time { return previewTime }
+	localZone = time.UTC
 	ui.NowFunc = func() time.Time { return previewTime }
 }
 
@@ -26,6 +27,7 @@ const (
 	PreviewWaitingForCutover PreviewType = "waiting_for_cutover"
 	PreviewCuttingOver       PreviewType = "cutting_over"
 	PreviewCompleted         PreviewType = "completed"
+	PreviewRetrying          PreviewType = "retrying"
 	PreviewFailed            PreviewType = "failed"
 	PreviewStopped           PreviewType = "stopped"
 	PreviewStates            PreviewType = "states"
@@ -143,7 +145,7 @@ const (
 	PreviewCommentApplyCompleted               PreviewType = "comment_apply_completed"                 // Apply completed (all tables done)
 	PreviewCommentApplyFailed                  PreviewType = "comment_apply_failed"                    // Apply failed (1 done, 1 failed, 1 cancelled)
 	PreviewCommentApplyFailedBeforeRowCopy     PreviewType = "comment_apply_failed_before_row_copy"    // Apply failed before row copy (preflight rejection, per-table error)
-	PreviewCommentApplyRetrying                PreviewType = "comment_apply_retrying"                  // Apply interrupted, retrying automatically (attempt counter)
+	PreviewCommentApplyRetrying                PreviewType = "comment_apply_retrying"                  // Apply interrupted, retrying automatically (attempt counter + next attempt time)
 	PreviewCommentApplyStopped                 PreviewType = "comment_apply_stopped"                   // Apply stopped (1 done, 1 stopped)
 	PreviewCommentApplyWaitingCutover          PreviewType = "comment_apply_waiting_cutover"           // Waiting for cutover (deferred, operator triggers)
 	PreviewCommentApplyWaitingCutoverAutomatic PreviewType = "comment_apply_waiting_cutover_automatic" // Waiting for cutover (non-deferred, drive triggers)

pkg/cmd/internal/templates/preview_comment.go

@@ -363,6 +363,7 @@ func previewCLIApplyAllOutput() {
 		// MySQL: single table
 		{"MYSQL: SINGLE TABLE RUNNING", previewProgressOutput},
 		{"MYSQL: SINGLE TABLE COMPLETED", previewCompletedOutput},
+		{"MYSQL: SINGLE TABLE RETRYING", previewRetryingOutput},
 		{"MYSQL: SINGLE TABLE FAILED", previewFailedOutput},
 		{"MYSQL: SINGLE TABLE STOPPED", previewStoppedOutput},
 		{"MYSQL: SINGLE TABLE WAITING FOR CUTOVER", previewWaitingForCutoverOutput},

pkg/cmd/internal/templates/preview_dispatch.go

@@ -29,6 +29,8 @@ func PreviewCLIOutput(previewType PreviewType) {
 		previewCuttingOverOutput()
 	case PreviewCompleted:
 		previewCompletedOutput()
+	case PreviewRetrying:
+		previewRetryingOutput()
 	case PreviewFailed:
 		previewFailedOutput()
 	case PreviewStopped:

pkg/cmd/internal/templates/preview_progress.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/block/schemabot/pkg/apitypes"
 	"github.com/block/schemabot/pkg/state"
+	"github.com/block/schemabot/pkg/storage"
 	"vitess.io/vitess/go/vt/key"
 )
 
@@ -872,6 +873,34 @@ func previewFailedOutput() {
 	WriteProgress(data)
 }
 
+// previewRetryingOutput shows an apply between automatic recovery attempts: the
+// drive was interrupted mid-copy, part of the retry budget is spent, and the
+// next attempt is armed for a fixed time. The Retry row is what tells an
+// operator this apply is waiting rather than wedged. The wait is the one the
+// real policy arms for this attempt, so the preview cannot drift from it.
+func previewRetryingOutput() {
+	data := ProgressData{
+		State:        state.Apply.FailedRetryable,
+		Engine:       "Spirit",
+		ApplyID:      "apply-a1b2c3d4e5f6",
+		StartedAt:    previewTime.Add(-4 * time.Minute).Format(time.RFC3339),
+		ErrorMessage: "connection reset by peer",
+		Attempt:      3,
+		RetryAfter:   previewTime.Add(storage.RetryBackoff(3)).Format(time.RFC3339),
+		Tables: []TableProgress{
+			{
+				TableName: "users", Namespace: "testapp",
+				DDL:             "ALTER TABLE `users` ADD INDEX `idx_email_created` (`email`, `created_at`)",
+				Status:          state.Task.FailedRetryable,
+				RowsCopied:      156342,
+				RowsTotal:       397453,
+				PercentComplete: 39,
+			},
+		},
+	}
+	WriteProgress(data)
+}
+
 func previewStoppedOutput() {
 	// Sample progress with stopped state (mid-apply stop)
 	startedAt := previewTime.Add(-3 * time.Minute).Format(time.RFC3339)