diff --git a/docs/move.md b/docs/move.md index fe9aa5e9..f25c527e 100644 --- a/docs/move.md +++ b/docs/move.md @@ -14,7 +14,7 @@ This will copy all tables from the source database to the target database, verif ## Configuration - [checkpoint-max-age](#checkpoint-max-age) -- [create-sentinel](#create-sentinel) +- [defer-cutover](#defer-cutover) - [defer-secondary-indexes](#defer-secondary-indexes) - [force](#force) - [reverse-window](#reverse-window) @@ -31,11 +31,11 @@ This will copy all tables from the source database to the target database, verif The maximum age of a checkpoint before Move refuses to resume from it. Replaying many days of accumulated binary logs can be slower than re-copying, and the binary logs may have been purged in the meantime. -Unlike [migrate](migrate.md#checkpoint-max-age), Move does **not** fall back to a fresh copy when the checkpoint is too old: the target tables already contain rows (which is why the resume path was selected), so silently restarting is not possible. Instead the move fails with a `checkpoint is too old to safely resume` error. To proceed, either re-run with a larger `--checkpoint-max-age`, or wipe the target tables (including the `_spirit_checkpoint` table) and restart the move from scratch. +Unlike [migrate](migrate.md#checkpoint-max-age), Move does **not** fall back to a fresh copy when the checkpoint is too old: the target tables already contain rows (which is why the resume path was selected), so silently restarting is not possible. Instead the move fails with a `checkpoint is too old to safely resume` error. To proceed, either re-run with a larger `--checkpoint-max-age`, or wipe the target tables (including the `_spirit_move_checkpoint` table) and restart the move from scratch. The same caveats about [resuming across Spirit binary versions](migrate.md#resuming-across-spirit-binary-versions) apply to Move, with one difference: where migrate silently discards an unreadable checkpoint and starts fresh, Move fails the run. -### create-sentinel +### defer-cutover - Type: Boolean - Default value: `false` @@ -44,12 +44,12 @@ When set to `true`, a sentinel table (`_spirit_sentinel`) is created on the firs #### Two-checksum model -When `create-sentinel` is in use Move runs two checksums: +When `defer-cutover` is in use Move runs two checksums: 1. The **initial checksum** runs after copy-rows completes and before Move starts waiting on the sentinel. This is the correctness gate; the cutover will not proceed unless the initial checksum succeeds. 2. The **continuous checksum** runs in a loop *while* Move is waiting on the sentinel to be dropped. It is a best-effort consistency re-check so that the data is re-verified close to the moment of cutover, even if the sentinel sits for hours. The continuous loop is interrupted as soon as the sentinel is dropped, and Move proceeds to cutover. One exception: if a pass had already detected a mismatch and is mid-recopy, the in-flight repair runs to completion (bounded by an internal per-chunk timeout) before cutover continues, since cancelling between the DELETE on targets and the re-apply from sources would leave the chunk inconsistent. A real repair error surfaced this way aborts the run instead of proceeding to cutover. -Move order (with `create-sentinel`): +Move order (with `defer-cutover`): ``` copy rows → initial checksum → wait on sentinel (continuous checksum loop) → cutover diff --git a/pkg/move/README.md b/pkg/move/README.md index bb1d1b62..b1b15a34 100644 --- a/pkg/move/README.md +++ b/pkg/move/README.md @@ -43,7 +43,7 @@ A run interrupted *before its first checkpoint dump* leaves the checkpoint table ### Sentinel Table -When `CreateSentinel` is enabled, the runner creates a `_spirit_sentinel` table on the first target (targets[0], alongside the checkpoint) during setup (before the copy starts) and then *blocks before cutover* until it is dropped by an external actor. The wait sits between the initial checksum and the cutover. This provides a coordination point for orchestration systems that need to perform additional steps between copy completion and cutover. +When `DeferCutOver` is enabled, the runner creates a `_spirit_sentinel` table on the first target (targets[0], alongside the checkpoint) during setup (before the copy starts) and then *blocks before cutover* until it is dropped by an external actor. The wait sits between the initial checksum and the cutover. This provides a coordination point for orchestration systems that need to perform additional steps between copy completion and cutover. While the sentinel blocks the cutover, the runner re-runs the checksum in a loop (the "continuous checksum") so that the data is re-verified close to the moment of cutover, even if the sentinel sits for hours. The first iteration starts one hour after the initial checksum, and subsequent iterations are capped at one per hour so that small tables do not churn the table lock back-to-back; the wait is interrupted when the sentinel is dropped. One exception: if a pass had already detected a mismatch and is mid-recopy, the in-flight repair runs to completion (bounded by an internal per-chunk timeout) before cutover continues, because the DELETE-from-targets + re-apply-from-sources pair must stay atomic. See [docs/move.md](../../docs/move.md) for the user-facing description. diff --git a/pkg/move/check/check.go b/pkg/move/check/check.go index 8fc772fb..cc753727 100644 --- a/pkg/move/check/check.go +++ b/pkg/move/check/check.go @@ -33,10 +33,10 @@ type SourceResource struct { // Resources contains the resources needed for move checks type Resources struct { - Sources []SourceResource - Targets []applier.Target - SourceTables []*table.TableInfo - CreateSentinel bool + Sources []SourceResource + Targets []applier.Target + SourceTables []*table.TableInfo + DeferCutOver bool // MoveEverything is true when no explicit table list was supplied (i.e. // move.SourceTables is empty), so every table in each source database is // being moved. The source_schema_consistency check uses this to decide diff --git a/pkg/move/checksum_invariant_test.go b/pkg/move/checksum_invariant_test.go index 56400ee4..85c9eaa4 100644 --- a/pkg/move/checksum_invariant_test.go +++ b/pkg/move/checksum_invariant_test.go @@ -71,11 +71,11 @@ func setupRunnerForChecksumTest(t *testing.T, dbSuffix string) (*Runner, context }) move := &Move{ - SourceDSN: sourceDSN, - TargetDSN: targetDSN, - Threads: 1, - WriteThreads: 1, - CreateSentinel: false, + SourceDSN: sourceDSN, + TargetDSN: targetDSN, + Threads: 1, + WriteThreads: 1, + DeferCutOver: false, } r, err := NewRunner(move) require.NoError(t, err) diff --git a/pkg/move/move.go b/pkg/move/move.go index c0a04efa..407fae7e 100644 --- a/pkg/move/move.go +++ b/pkg/move/move.go @@ -20,7 +20,7 @@ type Move struct { TargetChunkSize uint64 `name:"target-chunk-size" help:"In-memory byte budget per copy chunk (in bytes)." default:"16777216"` Threads int `name:"threads" help:"How many chunks to copy in parallel" default:"2"` WriteThreads int `name:"write-threads" help:"How many concurrent write threads to use per target" default:"4"` - CreateSentinel bool `name:"create-sentinel" help:"Create a sentinel table on the first target database to block after table copy" default:"false"` + DeferCutOver bool `name:"defer-cutover" help:"Defer cutover (and continuous checksum) until the sentinel table on the first target database is dropped" default:"false"` DeferSecondaryIndexes bool `name:"defer-secondary-indexes" help:"Create target tables without secondary indexes, add them before cutover" default:"false"` CheckpointMaxAge time.Duration `name:"checkpoint-max-age" help:"Maximum age of a checkpoint before refusing to resume from it" optional:"" default:"168h"` // Force makes the runner wipe the target tables and start the copy fresh when diff --git a/pkg/move/move_sharded_test.go b/pkg/move/move_sharded_test.go index 6ec34ab9..c4a72d81 100644 --- a/pkg/move/move_sharded_test.go +++ b/pkg/move/move_sharded_test.go @@ -389,7 +389,7 @@ func TestShardedMoveVindexUpdateFails(t *testing.T) { WriteThreads: 2, // The sentinel blocks the move before cutover, giving the test a // deterministic window in which the repl client is streaming. - CreateSentinel: true, + DeferCutOver: true, ShardingProvider: &testShardingProvider{ shardingColumn: "user_id", hashFunc: testutils.EvenOddHasher, diff --git a/pkg/move/move_test.go b/pkg/move/move_test.go index 1f00189f..ebb27fe1 100644 --- a/pkg/move/move_test.go +++ b/pkg/move/move_test.go @@ -52,11 +52,11 @@ func TestBasicMove(t *testing.T) { // test move := &Move{ - SourceDSN: sourceDSN, - TargetDSN: targetDSN, - Threads: 2, - WriteThreads: 2, - CreateSentinel: false, + SourceDSN: sourceDSN, + TargetDSN: targetDSN, + Threads: 2, + WriteThreads: 2, + DeferCutOver: false, } require.NoError(t, move.Run()) } @@ -192,11 +192,11 @@ func TestEmptyDatabaseMove(t *testing.T) { // Run move with empty source move := &Move{ - SourceDSN: sourceDSN, - TargetDSN: targetDSN, - Threads: 4, - WriteThreads: 4, - CreateSentinel: false, + SourceDSN: sourceDSN, + TargetDSN: targetDSN, + Threads: 4, + WriteThreads: 4, + DeferCutOver: false, } runner, err := NewRunner(move) @@ -253,11 +253,11 @@ func TestMoveReservedWordPK(t *testing.T) { ") ENGINE=InnoDB") move := &Move{ - SourceDSN: sourceDSN, - TargetDSN: targetDSN, - Threads: 2, - WriteThreads: 2, - CreateSentinel: false, + SourceDSN: sourceDSN, + TargetDSN: targetDSN, + Threads: 2, + WriteThreads: 2, + DeferCutOver: false, } require.NoError(t, move.Run()) } @@ -297,11 +297,11 @@ func TestMoveReservedWordTableName(t *testing.T) { ") ENGINE=InnoDB") move := &Move{ - SourceDSN: sourceDSN, - TargetDSN: targetDSN, - Threads: 2, - WriteThreads: 2, - CreateSentinel: false, + SourceDSN: sourceDSN, + TargetDSN: targetDSN, + Threads: 2, + WriteThreads: 2, + DeferCutOver: false, } require.NoError(t, move.Run()) } diff --git a/pkg/move/runner.go b/pkg/move/runner.go index b6a89b11..ab8b84b8 100644 --- a/pkg/move/runner.go +++ b/pkg/move/runner.go @@ -950,9 +950,14 @@ func (r *Runner) newCopy(ctx context.Context) error { // Create the sentinel on targets[0], alongside the checkpoint, so all of // move's coordination tables live in one place (the source tables are - // renamed out of the way at cutover). Idempotent (CREATE IF NOT EXISTS) so a - // resume recreates it and a concurrent existence probe never sees it absent. - if r.move.CreateSentinel { + // renamed out of the way at cutover). Only the fresh-copy path creates it; + // a resume never does, and does not need to — the sentinel lives on the + // target, so it simply survives, and the existence-driven sentinel.Wait + // below blocks again. (If the operator dropped it before the resume, the + // resumed move cuts over without waiting, matching migrate.) Creation is + // idempotent (CREATE IF NOT EXISTS) so that a concurrent existence probe + // never sees it absent — see TestCreateSentinelTableIdempotent. + if r.move.DeferCutOver { if err := sentinel.Create(ctx, r.targets[0].DB); err != nil { return err } @@ -1593,7 +1598,7 @@ func (r *Runner) runChecks(ctx context.Context, scope check.ScopeFlag) error { Sources: sources, Targets: r.targets, SourceTables: r.sourceTables, - CreateSentinel: r.move.CreateSentinel, + DeferCutOver: r.move.DeferCutOver, MoveEverything: len(r.move.SourceTables) == 0, }, r.logger, scope) } @@ -1706,7 +1711,7 @@ func (r *Runner) restoreIndexesForTargets(ctx context.Context, host string, targ // postCopyPhase runs the work that happens between copy-rows and the // sentinel wait: drain the binlog backlog, restore secondary indexes // (if deferred), run ANALYZE TABLE, and perform the initial checksum. -// When create-sentinel is not in use this is also the last phase +// When defer-cutover is not in use this is also the last phase // before cutover. func (r *Runner) postCopyPhase(ctx context.Context) error { // Flush all pending events, but leave the periodic flush running until diff --git a/pkg/move/runner_test.go b/pkg/move/runner_test.go index 2ddca0a1..e1c6516d 100644 --- a/pkg/move/runner_test.go +++ b/pkg/move/runner_test.go @@ -134,7 +134,7 @@ func testMoveWithConcurrentWrites(t *testing.T, deferSecondaryIndexes bool) { TargetDSN: targetDSN, Threads: 2, WriteThreads: 2, - CreateSentinel: false, + DeferCutOver: false, DeferSecondaryIndexes: deferSecondaryIndexes, } @@ -312,11 +312,11 @@ func TestMoveWithNewTableCreation(t *testing.T) { // it has a sentinel so it will never complete accidentally time.Sleep(100 * time.Millisecond) move := Move{ - SourceDSN: sourceDSN, - TargetDSN: targetDSN, - Threads: 2, - WriteThreads: 2, - CreateSentinel: true, + SourceDSN: sourceDSN, + TargetDSN: targetDSN, + Threads: 2, + WriteThreads: 2, + DeferCutOver: true, } wg.Go(func() { err = move.Run() @@ -404,11 +404,11 @@ func TestMoveFailsGracefullyWithMinimalRBR(t *testing.T) { }) move := &Move{ - SourceDSN: sourceDSN, - TargetDSN: targetDSN, - Threads: 2, - WriteThreads: 2, - CreateSentinel: false, + SourceDSN: sourceDSN, + TargetDSN: targetDSN, + Threads: 2, + WriteThreads: 2, + DeferCutOver: false, } err = move.Run() @@ -825,11 +825,11 @@ func TestMoveWithVarcharPK(t *testing.T) { time.Sleep(100 * time.Millisecond) move := &Move{ - SourceDSN: sourceDSN, - TargetDSN: targetDSN, - Threads: 2, - WriteThreads: 2, - CreateSentinel: false, + SourceDSN: sourceDSN, + TargetDSN: targetDSN, + Threads: 2, + WriteThreads: 2, + DeferCutOver: false, } err = move.Run() cancel() diff --git a/pkg/move/sentinel_test.go b/pkg/move/sentinel_test.go index 77af4193..6c94d5e0 100644 --- a/pkg/move/sentinel_test.go +++ b/pkg/move/sentinel_test.go @@ -26,7 +26,7 @@ func sentinelTestTableExists(t *testing.T, db *sql.DB, schema, name string) bool return true } -// TestMoveSentinelDropReleasesCutover: with --create-sentinel, dropping the +// TestMoveSentinelDropReleasesCutover: with --defer-cutover, dropping the // sentinel must RELEASE the cutover and let the move finish — not be seen as a // schema change that cancels it. The sentinel lives on targets[0], so the drop // is a target-side DDL that the source-watching change feed must ignore. Uses a @@ -52,11 +52,11 @@ func TestMoveSentinelDropReleasesCutover(t *testing.T) { defer utils.CloseAndLog(ctl) m := &Move{ - SourceDSN: src.FormatDSN(), - TargetDSN: dst.FormatDSN(), - Threads: 1, - WriteThreads: 1, - CreateSentinel: true, + SourceDSN: src.FormatDSN(), + TargetDSN: dst.FormatDSN(), + Threads: 1, + WriteThreads: 1, + DeferCutOver: true, } runner, err := NewRunner(m) require.NoError(t, err)