progress: strategy-wide machine-readable execution progress

.golangci.yml

@@ -22,7 +22,27 @@ linters:
     - containedctx    # no context.Context stored in struct fields
     - sloglint        # structured-logging hygiene: static messages, snake_case keys
     - forbidigo       # no printing to process stdout; output goes to the injected writer
+    - depguard        # pin the TCB import boundary (SAFETY.md core dependency list)
   settings:
+    depguard:
+      rules:
+        # The safety-critical core may import only the recorded core dependency
+        # list (SAFETY.md): stdlib, pgx/v5, the parse boundary, and the other
+        # core packages. Anything else needs a recorded decision there first.
+        tcb:
+          files:
+            - "**/pkg/dbconn/**"
+            - "**/pkg/preflight/**"
+            - "**/pkg/executor/**"
+            - "**/pkg/progress/**"
+            - "!$test"
+          allow:
+            - $gostd
+            - github.com/jackc/pgx/v5
+            - github.com/block/pg-sprite/pkg/dbconn
+            - github.com/block/pg-sprite/pkg/preflight
+            - github.com/block/pg-sprite/pkg/progress
+            - github.com/block/pg-sprite/pkg/statement
     sloglint:
       static-msg: true
       key-naming-case: snake

SAFETY.md

@@ -31,7 +31,7 @@ The invariant registry (invariant IDs referenced below) lives in
 | `pkg/verdict` — structured outcome contract, rendering, exit codes | ❌ periphery | exists (Phase 1) | — |
 | `pkg/diffplan` — desired schema → routed convergence plan, the declarative front door as a library (the CLI `diff` and embedding orchestrators share it) | ❌ periphery | exists | — |
 | `internal/cli` — CLI, flags, help, prompts | ❌ periphery | `migrate`, `status`, `diff`, `fmt`, `lint`, and `suggest` exist | — |
-| status / progress / advisory rendering, metrics | ❌ periphery | planned | — |
+| `pkg/progress` — strategy-wide progress snapshots; the executors' observation seam (core imports it, so its locking discipline is core-critical); copy counters reserved for later | ✅ core | native progress exists | — |
 | orchestrator adapter | ❌ periphery | planned (Phase 11) | OC-* hold *at* the boundary |
 | `internal/testutil` | ❌ test-only | exists | — |
 
@@ -62,6 +62,12 @@ The short version — the full rules live in [docs/tcb-model.md](docs/tcb-model.
   `pgx/v5`, the parse boundary (`pkg/statement` → `wasilibs/go-pgquery`, the real PostgreSQL
   grammar — the native executor re-verifies statement shape itself rather than trusting the
   caller's classification; the grammar is load-bearing expertise, not copyable mechanics),
+  `pkg/progress` (the executors' progress-observation seam: they write state into a
+  caller-owned tracker whose mutators take only a memory lock, and its polling reads ride
+  the reserved verdict session behind a separate poll lock — the executor's own state
+  updates never wait for a database read, but the verdict handoff *is* observer-gated:
+  `StopConcurrentBuild` deliberately drains an in-flight poll before the executor reclaims
+  the session, a wait bounded by the poller's context and the session's `statement_timeout`),
   stdlib. The future decode path will add `pglogrepl`. Adding one requires a recorded decision (see the rubric in
   [docs/tcb-model.md](docs/tcb-model.md) — copy small things, take pinned dependencies only
   for load-bearing expertise).

docs/README.md

@@ -36,6 +36,7 @@ Aurora-only. Why that combination is the product is [vision.md](vision.md); star
 | [limitations.md](limitations.md) | The **current limitations** — schema changes pg-sprite refuses today, why they are unsafe or unsupported, and where an operator must act outside the engine. |
 | [lint-report.md](lint-report.md) | The **lint report contract** — the versioned JSON shape `pg-sprite lint` emits for offline CI gating: finding fields (verbatim SQL, line/column), the codes table, severities and exit behavior, the offline-conservatism rules, and how the contract versions relative to the plan report. |
 | [suggest-report.md](suggest-report.md) | The **suggest report contract** — the versioned JSON shape `pg-sprite suggest` emits for offline advice: the typed caveat vocabulary (what changes about how you must run a safer form, and what a failed step leaves behind), the typed guidance codes for rewrites the planner cannot construct, and the operation → safer form → caveats table (pinned by test). |
+| [progress-report.md](progress-report.md) | The **progress report contract** — the versioned JSON snapshot a caller receives when polling a running change through the `*WithProgress` entry points: phases and operations vocabularies, the terminal-freeze rule, server-observed work counters, and polling semantics (pinned by test). |
 | [engine-role.md](engine-role.md) | The **engine-role provisioning contract** — the tiered minimum access a PostgreSQL user needs to run schema changes against tables it does not own: role membership for owner-gated DDL, schema `CREATE` for index builds and shadow objects, `SET ROLE` for owner-correct shadow creation, replication access for CDC, and the explicit list of powers the engine role must *not* have. Preflight refusals name the missing `GRANT` and point here. |
 | [invalid-index-recovery.md](invalid-index-recovery.md) | The **operator runbook** for the one native-path outcome that needs a human — an invalid index the executor found or may have left. What each typed state licenses: when `DROP INDEX CONCURRENTLY` is proven safe, when the entry may be another actor's healthy in-flight build, and what to check when the executor could prove nothing. |
 | [testing.md](testing.md) | The **test-suite guide** — how to run the suite (unit, per-major, all supported majors, compose database), current coverage, the remaining executor-phase test obligations, and the vanilla-PostgreSQL-matrix vs real-Aurora validation boundary. |

docs/architecture.md

@@ -136,6 +136,7 @@ different levels of commitment:
 | `pkg/diffplan` | The declarative front door as a library: desired schema in, routed `plan.Report` out — the CLI `diff` and embedding orchestrators share this one pipeline | exists |
 | `pkg/router` | Route classified statements to native / copy-and-swap / refuse dispositions; copy-and-swap reports unavailable until that backend lands | exists (Phase 2.4) |
 | `pkg/executor` | Bounded optimistic native attempt, the concurrent index build, and the autocommit safer-sequence runner, with stable outcome codes; the full `Executor` contract (`Plan`/`Execute`/`Status`/`Abort`) arrives with the copy-and-swap backend | native execution exists |
+| `pkg/progress` | Strategy-wide, pollable progress snapshots: native phase/elapsed time, sequence position, retry attempt, and server-reported concurrent-index work; optional copy counters are reserved for copy-and-swap | native progress exists |
 | `pkg/table` | PK-range chunkers (single-column fast path, composite), dynamic time-based sizing | Phase 4 |
 | `pkg/copier` | Parallel chunked copy into the shadow table (never overwrites) | Phase 4 |
 | `pkg/checksum` | The mandatory correctness gate; continuous checker; repair primitive | Phase 5 |

docs/low-level-design.md

@@ -811,12 +811,14 @@ At the library seam, each executor outcome maps to a stable string code
 embedding `pkg/executor` branches on one vocabulary; the CLI's verdict JSON carries the same
 codes — an execution failure ends in a `failed` verdict (exit 1, distinct from the refusal
 exit 2) with the code, the failed step, and the committed prefix in `executed_sql`, so
-automation can distinguish nothing-committed from partial state left behind. Remaining
-Phase 3 work, roughly in order:
-
-- bound lock acquisition with timeout and retry for the blocking idioms,
-- progress reporting (`pg_stat_progress_create_index` by the build's backend PID, which the
-  executor already captures for its ownership proof).
+automation can distinguish nothing-committed from partial state left behind. Native execution
+exposes a caller-owned `progress.Tracker`. Embedders run a blocking executor
+call in their own bounded task and poll `Tracker.Progress(ctx)`: sequence position and elapsed
+time come from in-process state, while an active concurrent index build is read on demand from
+`pg_stat_progress_create_index` by the build's backend PID. There is no background poller to
+own or stop, and a missing progress-view row is represented as an inactive observation rather
+than an error. The same snapshot already reserves optional row and byte copy counters for the
+copy-and-swap backend.
 
 The copy-and-swap backend, including change capture, copying, applying, checksumming, and
 cutover, follows Phase 3.

docs/progress-report.md

@@ -0,0 +1,117 @@
+# The progress report contract
+
+The progress snapshot is the machine-readable observation a caller receives when it polls a
+running schema change through the `*WithProgress` executor entry points. It is the one JSON
+shape an operator or orchestrator consumes to display or act on execution progress. This
+document is the contract: the fields, the closed vocabularies, and the behavior required of
+a consumer. The Go source of truth is `pkg/progress`; `TestSnapshotJSONShape` pins the exact
+keys, including the example at the end of this page.
+
+## Versioning: `format_version`
+
+Every snapshot carries `format_version`. A consumer that does not recognize the version must
+**reject the snapshot** — never guess at field semantics. The version covers more than the
+field shape: the closed vocabularies below (phases, operations) are pinned to it. Adding a
+phase or operation value is a contract change and bumps `format_version`, even if no field
+is added or renamed.
+
+The [plan report](plan-report.md), [lint report](lint-report.md), and
+[suggest report](suggest-report.md) are separate contracts with their own `format_version`;
+all version independently.
+
+## Consumer behavior for unknown values
+
+`phase` and `detail.operation` draw from the closed vocabularies below. A consumer that
+meets a value it does not recognize must treat the execution's state as **unknown** — never
+map it onto a known value and proceed. Progress is observational: an unknown value never
+licenses a consumer to intervene in the change itself.
+
+## Snapshot fields
+
+| Field | Type | Presence | Meaning |
+|---|---|---|---|
+| `format_version` | int | always | Contract version; reject unknown versions. |
+| `phase` | string | always | Overall execution phase (see Phases). |
+| `step` | int | after the first step starts | 1-based position in a multi-step sequence. Absent before execution reaches step 1. |
+| `total_steps` | int | after `Start` | Number of steps in the execution; `1` for single-statement entry points. |
+| `elapsed_ns` | int | always | Nanoseconds since execution started. For a terminal phase, **frozen** at the instant the outcome was recorded — a late poll reports the execution's duration, not the observation's age. |
+| `step_elapsed_ns` | int | always | Nanoseconds since the current step started; frozen the same way at a terminal phase. |
+| `detail` | object | always | The operation currently executing (below). |
+
+## Detail fields
+
+| Field | Type | Presence | Meaning |
+|---|---|---|---|
+| `operation` | string | once execution starts | The current operation's execution class (see Operations). |
+| `server_phase` | string | active concurrent build only | PostgreSQL's own phase string from `pg_stat_progress_create_index`, verbatim. |
+| `active` | bool | always | Whether an operation is executing now. `false` with `phase: "running"` means a concurrent build's progress row has left the server view. |
+| `attempt` | int | bounded retries only | The current attempt number when the executor is inside its bounded retry loop. |
+| `work` | object | server-observed work only | Present exactly when the server published a progress row; then **every** counter below is present, so a fresh build reports honest zeros rather than an empty object. |
+
+### Work counters
+
+`blocks_done` / `blocks_total` and `tuples_done` / `tuples_total` come from
+`pg_stat_progress_create_index` during a concurrent index build. `rows_copied` /
+`rows_total` and `bytes_copied` / `bytes_total` are reserved for copy-and-swap and are `0`
+on every native operation — the engine never fabricates copy counters.
+
+## Phases
+
+| Value | Meaning |
+|---|---|
+| `pending` | Execution has not started. |
+| `running` | Execution is active. |
+| `finished` | Terminal: completed successfully. |
+| `failed` | Terminal: reached a terminal failure. |
+
+A terminal snapshot is immutable: once `finished` or `failed` is observed, every later poll
+returns the identical snapshot, elapsed values included.
+
+## Operations
+
+| Value | Meaning |
+|---|---|
+| `admitting` | A sequence's steps are still being validated; no statement has run yet. |
+| `optimistic` | One bounded direct native attempt. |
+| `brief` | A brief transactional sequence step. |
+| `validate-constraint` | A constraint-validation scan. |
+| `concurrent-index-build` | A concurrent index build (the one operation with server-observed `work`). |
+
+## Polling semantics
+
+The tracker is caller-owned and has no goroutines or timers: polling lifetime is exactly the
+caller's context. A poll during an active concurrent index build performs one read of the
+server's progress view over the executor's reserved session; every other poll is pure
+memory. On a query error the returned snapshot still carries the last-known tracker state —
+`phase` is never empty — with the error returned alongside for the caller to classify.
+
+## Example
+
+A poll during step 2 of a 3-step sequence, mid concurrent index build:
+
+```json
+{
+  "format_version": 1,
+  "phase": "running",
+  "step": 2,
+  "total_steps": 3,
+  "elapsed_ns": 2750000000,
+  "step_elapsed_ns": 750000000,
+  "detail": {
+    "operation": "concurrent-index-build",
+    "server_phase": "building index",
+    "active": true,
+    "attempt": 2,
+    "work": {
+      "rows_copied": 0,
+      "rows_total": 0,
+      "bytes_copied": 0,
+      "bytes_total": 0,
+      "blocks_done": 11,
+      "blocks_total": 40,
+      "tuples_done": 7,
+      "tuples_total": 21
+    }
+  }
+}
+```

docs/tcb-model.md

@@ -246,9 +246,10 @@ The AI-assistance posture differs per side of the boundary:
    bitcoin-core discipline).
 2. **`cutover`/`swap` API takes domain types only** — the `VerifiedShadow`/`CleanWatermark`/
    `TableLock` types land with their producing packages (Phases 4–7), not retrofitted.
-3. **The `// INV: <id>` convention has landed; `ErrInvariantViolation`** lands with the
+3. **The `// INV: <id>` convention and `ErrInvariantViolation` have landed** with the
    executor phases.
-4. **The enforcement backlog:** [SAFETY.md](../SAFETY.md) + depguard + CODEOWNERS, the
-   property/fuzz suites per rung above, and the optional TLA+ models for cutover and resume.
+4. **The enforcement backlog:** [SAFETY.md](../SAFETY.md), the depguard import-boundary
+   rule (`.golangci.yml`), and CODEOWNERS have landed; still open are the property/fuzz
+   suites per rung above and the optional TLA+ models for cutover and resume.
 5. **The periphery stays free.** None of this doc applies review friction to status text, CLI
    help, or docs — that's the point of having a boundary.

pkg/dbconn/dbconn.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
+	"errors"
 	"fmt"
 	"log/slog"
 	"os"
@@ -116,6 +117,37 @@ func ServerMajor(ctx context.Context, pool *pgxpool.Pool) (int, error) {
 	return major, nil
 }
 
+// IndexBuildProgress is one server observation of a concurrent index build.
+type IndexBuildProgress struct {
+	Phase       string
+	BlocksDone  uint64
+	BlocksTotal uint64
+	TuplesDone  uint64
+	TuplesTotal uint64
+}
+
+// RowQuerier is the session capability needed for a progress observation.
+type RowQuerier interface {
+	QueryRow(context.Context, string, ...any) pgx.Row
+}
+
+// ConcurrentIndexProgress reads the active build owned by backendPID. The
+// boolean is false when PostgreSQL has not published the row yet or the build
+// has already left the progress view.
+func ConcurrentIndexProgress(ctx context.Context, session RowQuerier, backendPID uint32) (IndexBuildProgress, bool, error) {
+	var p IndexBuildProgress
+	err := session.QueryRow(ctx, `SELECT phase, blocks_done, blocks_total, tuples_done, tuples_total
+		FROM pg_catalog.pg_stat_progress_create_index WHERE pid = $1`, backendPID).
+		Scan(&p.Phase, &p.BlocksDone, &p.BlocksTotal, &p.TuplesDone, &p.TuplesTotal)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return p, false, nil
+	}
+	if err != nil {
+		return p, false, fmt.Errorf("read concurrent index progress for backend %d: %w", backendPID, err)
+	}
+	return p, true, nil
+}
+
 // buildPoolConfig translates Config into a pgxpool configuration. It is pure
 // (no dialing), so every option's wiring is unit-testable without a server.
 func buildPoolConfig(cfg Config) (*pgxpool.Config, error) {

pkg/dbconn/dbconn_integration_test.go

@@ -40,6 +40,13 @@ func TestPoolIntegration(t *testing.T) {
 		assert.Equal(t, want, version)
 	})
 
+	t.Run("absent concurrent index progress is not an error", func(t *testing.T) {
+		observation, active, err := dbconn.ConcurrentIndexProgress(t.Context(), pool, 0)
+		require.NoError(t, err)
+		assert.False(t, active)
+		assert.Empty(t, observation.Phase)
+	})
+
 	t.Run("statement_timeout cancels runaway work", func(t *testing.T) {
 		_, err := pool.Exec(t.Context(), "SELECT pg_sleep(5)")
 		require.Error(t, err)

pkg/executor/native.go

@@ -24,6 +24,7 @@ import (
 	"github.com/jackc/pgx/v5/pgconn"
 	"github.com/jackc/pgx/v5/pgxpool"
 
+	"github.com/block/pg-sprite/pkg/progress"
 	"github.com/block/pg-sprite/pkg/statement"
 )
 
@@ -266,6 +267,22 @@ func (e *InvalidIndexError) Unwrap() []error {
 // whole budget, while a concurrent build takes only SHARE UPDATE EXCLUSIVE —
 // long builds on large tables are its purpose.
 func BuildIndexConcurrently(ctx context.Context, pool *pgxpool.Pool, sql string, b ConcurrentBudget) (IndexBuildReport, error) {
+	return buildIndexConcurrently(ctx, pool, sql, b, nil)
+}
+
+// BuildIndexConcurrentlyWithProgress runs a concurrent build while updating
+// tracker. The caller may poll tracker concurrently with this blocking call.
+func BuildIndexConcurrentlyWithProgress(ctx context.Context, pool *pgxpool.Pool, sql string, b ConcurrentBudget, tracker *progress.Tracker) (rep IndexBuildReport, err error) {
+	if tracker == nil {
+		return rep, fmt.Errorf("%w: progress tracker is required", ErrInvariantViolation)
+	}
+	tracker.Start(1, progress.OperationConcurrentIndex)
+	tracker.StartStep(1, progress.OperationConcurrentIndex)
+	defer func() { tracker.Finish(err) }()
+	return buildIndexConcurrently(ctx, pool, sql, b, tracker)
+}
+
+func buildIndexConcurrently(ctx context.Context, pool *pgxpool.Pool, sql string, b ConcurrentBudget, tracker *progress.Tracker) (IndexBuildReport, error) {
 	var rep IndexBuildReport
 	if err := b.validate(); err != nil {
 		return rep, err
@@ -323,10 +340,19 @@ func BuildIndexConcurrently(ctx context.Context, pool *pgxpool.Pool, sql string,
 	// The backend PID anchors the post-failure ownership proof: recovery
 	// waits for this backend to stop before trusting the catalog.
 	pid := conn.Conn().PgConn().PID()
+	if tracker != nil {
+		tracker.SetConcurrentBuild(verdictConn, pid)
+	}
 
 	start := time.Now()
+	if tracker != nil {
+		start = tracker.Now()
+	}
 	_, buildErr := conn.Exec(ctx, sql)
-	elapsed := time.Since(start)
+	elapsed := elapsedSince(tracker, start)
+	if tracker != nil {
+		tracker.StopConcurrentBuild()
+	}
 	if buildErr == nil {
 		return verifiedBuildReport(ctx, conn, build, target, elapsed)
 	}