feat(github): revalidate stored check state at merge-queue admission

pkg/metrics/metrics.go

@@ -1640,6 +1640,21 @@ func RecordMergeGatePlanTimeHold(ctx context.Context, database, environment stri
 	)
 }
 
+// RecordMergeGroupAdmissionBlocked counts merge-queue admission checks posted
+// blocking: a queued pull request's stored check state turned blocking after
+// it entered the queue — most often a preflight hold from a sibling change's
+// in-flight apply on a shared target. The blocked PR leaves the queue and its
+// author re-queues once its check re-plans green, so an occasional count is
+// the gate working; a sustained rate on one repository means changes keep
+// queueing against a busy target — check the target's apply activity.
+func RecordMergeGroupAdmissionBlocked(ctx context.Context, repository, environment string) {
+	addCounter(ctx, "schemabot.merge_gate.merge_group_admission_blocked_total",
+		"Total merge-queue admission checks posted blocking because the queued PR's stored check state blocks", "{check}",
+		attribute.String("repository", repository),
+		EnvironmentAttribute(environment),
+	)
+}
+
 // RecordMergeGateTerminatedStuck counts merge gate requests terminated
 // by the stuck-processing sweep: rows wedged past the attempt cap with an
 // expired lease (a driver hard-killed on its final attempt). Each terminated

pkg/webhook/check_aggregate.go

@@ -197,6 +197,22 @@ func hasBlockingCheckForEnvironment(checks []*storage.Check, environment string)
 	return false
 }
 
+// blockingChecksForEnvironment returns the stored rows that block a passing
+// aggregate for the environment. Callers list them on operator-facing
+// surfaces so a block names the databases behind it.
+func blockingChecksForEnvironment(checks []*storage.Check, environment string) []*storage.Check {
+	var blocking []*storage.Check
+	for _, c := range checks {
+		if environment != aggregateSentinel && c.Environment != environment {
+			continue
+		}
+		if checkBlocksPassingAggregate(c) {
+			blocking = append(blocking, c)
+		}
+	}
+	return blocking
+}
+
 // awaitingCurrentCommitTitle is the aggregate title shown when only results
 // recorded for another commit hold the aggregate open — nothing is running for
 // the commit the Check Run is published on, and the current commit's own plan

pkg/webhook/durable_merge_group_test.go

@@ -21,7 +21,7 @@ import (
 func durableMergeGroupEvent() *storage.WebhookEvent {
 	payload := []byte(`{
 		"action": "checks_requested",
-		"merge_group": {"head_sha": "mergesha123"},
+		"merge_group": {"head_sha": "mergesha123", "head_ref": "refs/heads/gh-readonly-queue/main/pr-1-mergesha123"},
 		"repository": {"full_name": "octocat/hello-world"},
 		"installation": {"id": 12345}
 	}`)
@@ -191,6 +191,132 @@ func TestDurableMergeGroupDriverParticipantStaysSilent(t *testing.T) {
 	}
 }
 
+// durableMergeGroupStorage layers a scripted check store over the durable
+// webhook harness so driver tests can seed stored check state for the
+// admission fold.
+type durableMergeGroupStorage struct {
+	durableWebhookTestStorage
+	checks storage.CheckStore
+}
+
+func (s *durableMergeGroupStorage) Checks() storage.CheckStore {
+	return s.checks
+}
+
+func newDurableMergeGroupDriverHandler(t *testing.T, store storage.WebhookEventStore, checks storage.CheckStore, config *api.ServerConfig, factory *fakeClientFactory) *Handler {
+	t.Helper()
+	logger := slog.New(slog.NewTextHandler(io.Discard, nil))
+	service := api.New(&durableMergeGroupStorage{
+		durableWebhookTestStorage: durableWebhookTestStorage{webhookEvents: store},
+		checks:                    checks,
+	}, config, nil, logger)
+	if factory == nil {
+		factory = &fakeClientFactory{}
+	}
+	return NewHandler(service, factory, nil, logger, WithDurableWebhookDispatch())
+}
+
+// When the queued PR's stored check state blocks — a sibling apply's preflight
+// hold landed after the PR entered the queue — the driver posts a blocking
+// admission check on the merge-group head instead of the pass the PR queued
+// with, and the delivery still completes: the verdict was delivered.
+func TestDurableMergeGroupDriverBlocksOnStoredHold(t *testing.T) {
+	client, mux := setupGitHubServer(t)
+	created := make(chan checkRunCapture, 10)
+	comments := make(chan string, 10)
+	mux.HandleFunc("POST /repos/octocat/hello-world/check-runs", func(w http.ResponseWriter, r *http.Request) {
+		var c checkRunCapture
+		require.NoError(t, json.NewDecoder(r.Body).Decode(&c))
+		created <- c
+		w.WriteHeader(http.StatusCreated)
+		_ = json.NewEncoder(w).Encode(map[string]any{"id": 556})
+	})
+	mux.HandleFunc("GET /repos/octocat/hello-world/issues/1/comments", func(w http.ResponseWriter, _ *http.Request) {
+		_ = json.NewEncoder(w).Encode([]map[string]any{})
+	})
+	mux.HandleFunc("POST /repos/octocat/hello-world/issues/1/comments", func(w http.ResponseWriter, r *http.Request) {
+		var c struct {
+			Body string `json:"body"`
+		}
+		require.NoError(t, json.NewDecoder(r.Body).Decode(&c))
+		comments <- c.Body
+		w.WriteHeader(http.StatusCreated)
+		_ = json.NewEncoder(w).Encode(map[string]any{"id": 777})
+	})
+	installClient := ghclient.NewInstallationClient(client, testLogger())
+
+	store := newScriptedWebhookEventStore(durableMergeGroupEvent())
+	checks := &foldCheckStore{byPR: []*storage.Check{{
+		Repository:   "octocat/hello-world",
+		PullRequest:  1,
+		HeadSHA:      "prhead123",
+		Environment:  "production",
+		DatabaseType: "mysql",
+		DatabaseName: "widgets",
+		Status:       "completed",
+		Conclusion:   "action_required",
+	}}}
+	config := &api.ServerConfig{
+		AllowedEnvironments: []string{"production"},
+		Repos:               map[string]api.RepoConfig{"octocat/hello-world": {}},
+	}
+	h := newDurableMergeGroupDriverHandler(t, store, checks, config, &fakeClientFactory{client: installClient})
+
+	h.driveNextDurableWebhook(t.Context(), 0, "test-host/1/webhook-driver-0")
+
+	select {
+	case c := <-created:
+		assert.Equal(t, "SchemaBot (production)", c.Name)
+		assert.Equal(t, "mergesha123", c.HeadSHA)
+		assert.Equal(t, "action_required", c.Conclusion)
+	case <-time.After(durableWebhookTestDeadline):
+		t.Fatal("expected the driver to post a blocking merge_group check")
+	}
+	select {
+	case body := <-comments:
+		assert.Contains(t, body, "Removed From Merge Queue")
+		assert.Contains(t, body, "`widgets` in `production`")
+		assert.Contains(t, body, mergeQueueEjectedCommentMarker("mergesha123"))
+	case <-time.After(durableWebhookTestDeadline):
+		t.Fatal("expected the driver to post the ejection guidance comment")
+	}
+	select {
+	case <-store.completed:
+	case <-time.After(durableWebhookTestDeadline):
+		t.Fatal("expected the delivery to be marked completed")
+	}
+	require.Empty(t, store.failed)
+}
+
+// A storage read failure during the admission fold is retryable: the verdict
+// cannot be computed from unknown state, so the delivery is retried rather
+// than completed with a guessed check.
+func TestDurableMergeGroupDriverRetriesStorageFailure(t *testing.T) {
+	client, _ := setupGitHubServer(t)
+	installClient := ghclient.NewInstallationClient(client, testLogger())
+
+	store := newScriptedWebhookEventStore(durableMergeGroupEvent())
+	checks := &foldCheckStore{byPRErr: errors.New("storage unavailable")}
+	config := &api.ServerConfig{
+		AllowedEnvironments: []string{"production"},
+		Repos:               map[string]api.RepoConfig{"octocat/hello-world": {}},
+	}
+	h := newDurableMergeGroupDriverHandler(t, store, checks, config, &fakeClientFactory{client: installClient})
+
+	before := time.Now()
+	h.driveNextDurableWebhook(t.Context(), 0, "test-host/1/webhook-driver-0")
+
+	select {
+	case failure := <-store.failed:
+		require.NotNil(t, failure.retryAfter, "storage failure must stay retryable")
+		require.True(t, failure.retryAfter.After(before), "retry must be scheduled in the future")
+		require.Contains(t, failure.errMsg, "storage unavailable")
+	default:
+		t.Fatal("expected storage failure to be marked failed")
+	}
+	require.Empty(t, store.completed)
+}
+
 // A GitHub client failure while posting the check is retryable: the merge queue
 // blocks until the check lands, so the delivery must be retried, not dropped.
 func TestDurableMergeGroupDriverRetriesClientFailure(t *testing.T) {