fix(op-supernode): wait for initial EL sync before cold start

op-node/node/node.go

@@ -1001,3 +1001,10 @@ func (n *OpNode) SyncStatus() *eth.SyncStatus {
 	}
 	return n.l2Driver.StatusTracker.SyncStatus()
 }
+
+func (n *OpNode) IsEngineInitialELSyncing() bool {
+	if n.l2Driver == nil || n.l2Driver.SyncDeriver == nil || n.l2Driver.SyncDeriver.Engine == nil {
+		return false
+	}
+	return n.l2Driver.SyncDeriver.Engine.IsEngineInitialELSyncing()
+}

op-supernode/supernode/activity/interop/algo_test.go

@@ -1073,9 +1073,10 @@ type algoMockChain struct {
 func (m *algoMockChain) BlockNumberToTimestamp(ctx context.Context, blocknum uint64) (uint64, error) {
 	return 0, nil
 }
-func (m *algoMockChain) FirstSafeHeadTimestamp(ctx context.Context) (uint64, error) {
+func (m *algoMockChain) FirstProvableSafeHeadTimestamp(ctx context.Context) (uint64, error) {
 	return 0, cc.ErrSafeDBEmpty
 }
+func (m *algoMockChain) IsEngineInitialELSyncing() bool                   { return false }
 func (m *algoMockChain) ID() eth.ChainID                                  { return m.id }
 func (m *algoMockChain) Start(ctx context.Context) error                  { return nil }
 func (m *algoMockChain) Stop(ctx context.Context) error                   { return nil }

op-supernode/supernode/activity/interop/interop_test.go

@@ -1751,12 +1751,14 @@ type mockChainContainer struct {
 	// by tests that exercise the genesis-clamp path in runLogBackfill.
 	blockNumberToTimestampOverride func(ctx context.Context, blocknum uint64) (uint64, error)
 
-	// firstSafeHeadTimestamp lets tests stub FirstSafeHeadTimestamp.
+	// firstSafeHeadTimestamp lets tests stub FirstProvableSafeHeadTimestamp.
 	// firstSafeHeadTimestampErr defaults to chain_container.ErrSafeDBEmpty
 	// when neither field is set so the cold-start init loop keeps waiting.
-	firstSafeHeadTimestamp    uint64
-	firstSafeHeadTimestampSet bool
-	firstSafeHeadTimestampErr error
+	firstSafeHeadTimestamp      uint64
+	firstSafeHeadTimestampSet   bool
+	firstSafeHeadTimestampErr   error
+	firstSafeHeadTimestampCalls int
+	initialELSyncing            bool
 }
 
 type invalidateBlockCall struct {
@@ -1790,9 +1792,10 @@ func (m *mockChainContainer) BlockNumberToTimestamp(ctx context.Context, blocknu
 	}
 	return 0, nil
 }
-func (m *mockChainContainer) FirstSafeHeadTimestamp(ctx context.Context) (uint64, error) {
+func (m *mockChainContainer) FirstProvableSafeHeadTimestamp(ctx context.Context) (uint64, error) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
+	m.firstSafeHeadTimestampCalls++
 	if m.firstSafeHeadTimestampErr != nil {
 		return 0, m.firstSafeHeadTimestampErr
 	}
@@ -1801,6 +1804,11 @@ func (m *mockChainContainer) FirstSafeHeadTimestamp(ctx context.Context) (uint64
 	}
 	return 0, cc.ErrSafeDBEmpty
 }
+func (m *mockChainContainer) IsEngineInitialELSyncing() bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.initialELSyncing
+}
 func (m *mockChainContainer) ELFinalizedHead(ctx context.Context) (eth.L2BlockRef, error) {
 	m.mu.Lock()
 	defer m.mu.Unlock()

op-supernode/supernode/activity/interop/log_backfill.go

@@ -11,11 +11,11 @@ import (
 )
 
 // advanceColdStartInit runs one best-effort pass at cold-start initialization:
-// it collects every chain's first SafeDB entry timestamp, picks
+// it collects every chain's first provable SafeDB entry timestamp, picks
 // verificationStartTimestamp = max(activation, max_c T_c), runs backfill, and
 // signals advance=true on success. Returns advance=false when any chain's
-// SafeDB is still empty (caller backs off and retries). Errors from the
-// backfill phase are fatal.
+// SafeDB is still missing a provable boundary or any VN is still in initial EL
+// sync (caller backs off and retries). Errors from the backfill phase are fatal.
 func (i *Interop) advanceColdStartInit() (bool, error) {
 	i.backfillAttempts.Add(1)
 
@@ -48,35 +48,45 @@ func (i *Interop) advanceColdStartInit() (bool, error) {
 }
 
 // collectFirstSafeHeadTimestamps queries every chain's SafeDB for its first
-// entry timestamp in parallel. Returns ready=false (without error) if any
-// chain has no entries yet; the caller backs off and retries. Other errors
-// are reported as-is.
+// provable entry timestamp in parallel. Returns ready=false (without error) if
+// any chain is still initial EL syncing or has no provable SafeDB boundary yet;
+// the caller backs off and retries. Other errors are reported as-is.
 func (i *Interop) collectFirstSafeHeadTimestamps() (map[eth.ChainID]uint64, bool, error) {
 	type res struct {
-		id  eth.ChainID
-		ts  uint64
-		err error
+		id               eth.ChainID
+		ts               uint64
+		initialELSyncing bool
+		err              error
 	}
 	results := make(chan res, len(i.chains))
 	for _, chain := range i.chains {
 		go func(c cc.ChainContainer) {
-			ts, err := c.FirstSafeHeadTimestamp(i.ctx)
+			if c.IsEngineInitialELSyncing() {
+				results <- res{id: c.ID(), initialELSyncing: true}
+				return
+			}
+			ts, err := c.FirstProvableSafeHeadTimestamp(i.ctx)
 			results <- res{id: c.ID(), ts: ts, err: err}
 		}(chain)
 	}
 	out := make(map[eth.ChainID]uint64, len(i.chains))
 	var firstErr error
-	emptyAny := false
+	waitAny := false
 	for range i.chains {
 		r := <-results
+		if r.initialELSyncing {
+			waitAny = true
+			i.log.Debug("interop cold start: chain is initial EL syncing, waiting", "chain", r.id)
+			continue
+		}
 		if r.err != nil {
 			if errors.Is(r.err, cc.ErrSafeDBEmpty) {
-				emptyAny = true
-				i.log.Debug("interop cold start: chain SafeDB empty, waiting", "chain", r.id)
+				waitAny = true
+				i.log.Debug("interop cold start: chain SafeDB boundary not provable, waiting", "chain", r.id)
 				continue
 			}
 			if firstErr == nil {
-				firstErr = fmt.Errorf("chain %s: first safe head timestamp: %w", r.id, r.err)
+				firstErr = fmt.Errorf("chain %s: first provable safe head timestamp: %w", r.id, r.err)
 			}
 			continue
 		}
@@ -85,7 +95,7 @@ func (i *Interop) collectFirstSafeHeadTimestamps() (map[eth.ChainID]uint64, bool
 	if firstErr != nil {
 		return nil, false, firstErr
 	}
-	if emptyAny {
+	if waitAny {
 		return nil, false, nil
 	}
 	return out, true, nil

op-supernode/supernode/activity/interop/startup_test.go

@@ -106,6 +106,31 @@ func TestAdvanceColdStartInit_WaitsWhenAnyChainEmpty(t *testing.T) {
 	require.False(t, h.interop.initialized.Load())
 }
 
+func TestAdvanceColdStartInit_WaitsWhenAnyChainInitialELSyncing(t *testing.T) {
+	var syncingChain *mockChainContainer
+	h := newInteropTestHarness(t).
+		WithActivation(1000).
+		WithChain(10, func(m *mockChainContainer) {
+			m.firstSafeHeadTimestamp = 1234
+			m.firstSafeHeadTimestampSet = true
+		}).
+		WithChain(20, func(m *mockChainContainer) {
+			m.initialELSyncing = true
+			m.firstSafeHeadTimestamp = 1500
+			m.firstSafeHeadTimestampSet = true
+			syncingChain = m
+		}).
+		Build()
+	h.interop.initialized.Store(false)
+	h.interop.verificationStartTimestamp = 0
+
+	advanced, err := h.interop.advanceColdStartInit()
+	require.NoError(t, err)
+	require.False(t, advanced, "must wait when any chain is still initial EL syncing")
+	require.False(t, h.interop.initialized.Load())
+	require.Zero(t, syncingChain.firstSafeHeadTimestampCalls, "must not use SafeDB history while EL sync may clear it")
+}
+
 // TestAdvanceColdStartInit_PicksMaxClampedToActivation: with all chains
 // reporting first SafeDB entries, verificationStartTimestamp is the max of
 // (activation, T_c).
@@ -150,7 +175,7 @@ func TestAdvanceColdStartInit_PicksMaxClampedToActivation(t *testing.T) {
 }
 
 // TestAdvanceColdStartInit_PropagatesNonEmptyErrors confirms that
-// FirstSafeHeadTimestamp errors other than ErrSafeDBEmpty are fatal.
+// FirstProvableSafeHeadTimestamp errors other than ErrSafeDBEmpty are fatal.
 func TestAdvanceColdStartInit_PropagatesNonEmptyErrors(t *testing.T) {
 
 	fault := errors.New("vn not running")

op-supernode/supernode/activity/supernode/supernode_test.go

@@ -114,9 +114,10 @@ func (m *mockCC) BlockNumberToTimestamp(ctx context.Context, blocknum uint64) (u
 	return 0, nil
 }
 
-func (m *mockCC) FirstSafeHeadTimestamp(ctx context.Context) (uint64, error) {
+func (m *mockCC) FirstProvableSafeHeadTimestamp(ctx context.Context) (uint64, error) {
 	return 0, cc.ErrSafeDBEmpty
 }
+func (m *mockCC) IsEngineInitialELSyncing() bool { return false }
 
 var _ cc.ChainContainer = (*mockCC)(nil)
 

op-supernode/supernode/activity/superroot/superroot_test.go

@@ -109,10 +109,11 @@ func (m *mockCC) TimestampToBlockNumber(ctx context.Context, ts uint64) (uint64,
 func (m *mockCC) BlockNumberToTimestamp(ctx context.Context, blocknum uint64) (uint64, error) {
 	return 0, nil
 }
-func (m *mockCC) FirstSafeHeadTimestamp(ctx context.Context) (uint64, error) {
+func (m *mockCC) FirstProvableSafeHeadTimestamp(ctx context.Context) (uint64, error) {
 	return 0, cc.ErrSafeDBEmpty
 }
-func (m *mockCC) Generation() uint64 { return 0 }
+func (m *mockCC) IsEngineInitialELSyncing() bool { return false }
+func (m *mockCC) Generation() uint64             { return 0 }
 
 var _ cc.ChainContainer = (*mockCC)(nil)
 

op-supernode/supernode/chain_container/chain_container.go

@@ -38,9 +38,9 @@ const virtualNodeVersion = "0.1.0"
 // retrying; recovery requires operator intervention.
 var ErrHistoryUnavailable = errors.New("safedb history unavailable on this node")
 
-// ErrSafeDBEmpty is returned by FirstSafeHeadTimestamp when SafeDB has no
-// entries yet. This is a transient condition during cold start while the VN
-// derives its first safe head; callers should back off and retry.
+// ErrSafeDBEmpty is returned by FirstProvableSafeHeadTimestamp when SafeDB has
+// no provable entries yet. This is a transient condition during cold start
+// while the VN derives safe heads; callers should back off and retry.
 var ErrSafeDBEmpty = errors.New("safedb has no entries yet")
 
 type ChainContainer interface {
@@ -55,10 +55,13 @@ type ChainContainer interface {
 	// TimestampToBlockNumber maps an L2 unix timestamp to the L2 block number (rollup derivation).
 	TimestampToBlockNumber(ctx context.Context, ts uint64) (uint64, error)
 	BlockNumberToTimestamp(ctx context.Context, blocknum uint64) (uint64, error)
-	// FirstSafeHeadTimestamp returns the L2 block timestamp of the first
-	// entry in this chain's SafeDB. Returns ErrSafeDBEmpty when the chain
-	// has not yet derived a safe head.
-	FirstSafeHeadTimestamp(ctx context.Context) (uint64, error)
+	// FirstProvableSafeHeadTimestamp returns the L2 block timestamp of the
+	// first SafeDB entry whose safe-head boundary can be resolved from retained
+	// SafeDB history. Returns ErrSafeDBEmpty until that boundary is available.
+	FirstProvableSafeHeadTimestamp(ctx context.Context) (uint64, error)
+	// IsEngineInitialELSyncing reports whether the embedded op-node is still
+	// in initial execution-layer sync.
+	IsEngineInitialELSyncing() bool
 	SyncStatus(ctx context.Context) (*eth.SyncStatus, error)
 	OptimisticAt(ctx context.Context, ts uint64) (l2, l1 eth.BlockID, err error)
 	// OutputRootAtL2BlockHash returns the L2 output root for the canonical
@@ -442,19 +445,27 @@ func (c *simpleChainContainer) BlockNumberToTimestamp(ctx context.Context, block
 	return c.vncfg.Rollup.TimestampForBlock(blocknum), nil
 }
 
-func (c *simpleChainContainer) FirstSafeHeadTimestamp(ctx context.Context) (uint64, error) {
+func (c *simpleChainContainer) FirstProvableSafeHeadTimestamp(ctx context.Context) (uint64, error) {
 	vn := c.getVN()
 	if vn == nil {
 		return 0, virtual_node.ErrVirtualNodeNotRunning
 	}
-	_, l2, err := vn.FirstSafeHeadEntry(ctx)
+	l2Num, err := vn.FirstProvableSafeHeadNumber(ctx)
 	if err != nil {
-		if errors.Is(err, safedb.ErrNotFound) {
+		if errors.Is(err, safedb.ErrNotFound) || errors.Is(err, virtual_node.ErrL1AtSafeHeadNotFound) {
 			return 0, ErrSafeDBEmpty
 		}
-		return 0, fmt.Errorf("first safedb entry: %w", err)
+		return 0, fmt.Errorf("first provable safedb entry: %w", err)
 	}
-	return c.BlockNumberToTimestamp(ctx, l2.Number)
+	return c.BlockNumberToTimestamp(ctx, l2Num)
+}
+
+func (c *simpleChainContainer) IsEngineInitialELSyncing() bool {
+	vn := c.getVN()
+	if vn == nil {
+		return false
+	}
+	return vn.IsEngineInitialELSyncing()
 }
 
 // LocalSafeBlockAtTimestamp returns the highest L2 block with timestamp <= ts using the L2 client,

op-supernode/supernode/chain_container/chain_container_test.go

@@ -111,9 +111,9 @@ func (m *mockVirtualNode) L1AtSafeHead(ctx context.Context, target eth.BlockID)
 	return m.safeHeadL1, m.safeHeadErr
 }
 
-// FirstSafeHeadEntry implements virtual_node.VirtualNode FirstSafeHeadEntry
-func (m *mockVirtualNode) FirstSafeHeadEntry(ctx context.Context) (eth.BlockID, eth.BlockID, error) {
-	return m.safeHeadL1, m.safeHeadL2, m.safeHeadErr
+// FirstProvableSafeHeadNumber implements virtual_node.VirtualNode FirstProvableSafeHeadNumber
+func (m *mockVirtualNode) FirstProvableSafeHeadNumber(ctx context.Context) (uint64, error) {
+	return m.safeHeadL2.Number, m.safeHeadErr
 }
 
 // LastL1 implements virtual_node.VirtualNode LastL1
@@ -136,6 +136,8 @@ func (m *mockVirtualNode) SyncStatus(ctx context.Context) (*eth.SyncStatus, erro
 	}, nil
 }
 
+func (m *mockVirtualNode) IsEngineInitialELSyncing() bool { return false }
+
 // SafeDB is not required by VirtualNode in these tests
 
 // mockEngineController is a mock implementation of engine_controller.EngineController
@@ -1055,12 +1057,13 @@ func (m *mockVNForL1AtSafeHeadError) SafeHeadAtL1(ctx context.Context, l1BlockNu
 func (m *mockVNForL1AtSafeHeadError) L1AtSafeHead(ctx context.Context, target eth.BlockID) (eth.BlockID, error) {
 	return eth.BlockID{}, m.l1AtSafeHeadErr
 }
-func (m *mockVNForL1AtSafeHeadError) FirstSafeHeadEntry(ctx context.Context) (eth.BlockID, eth.BlockID, error) {
-	return eth.BlockID{}, eth.BlockID{}, nil
+func (m *mockVNForL1AtSafeHeadError) FirstProvableSafeHeadNumber(ctx context.Context) (uint64, error) {
+	return 0, nil
 }
 func (m *mockVNForL1AtSafeHeadError) SyncStatus(ctx context.Context) (*eth.SyncStatus, error) {
 	return m.syncStatusResult, nil
 }
+func (m *mockVNForL1AtSafeHeadError) IsEngineInitialELSyncing() bool { return false }
 
 var _ virtual_node.VirtualNode = (*mockVNForL1AtSafeHeadError)(nil)
 

op-supernode/supernode/chain_container/invalidation_test.go

@@ -347,12 +347,13 @@ func (m *mockVNForInvalidation) SafeHeadAtL1(ctx context.Context, l1BlockNum uin
 func (m *mockVNForInvalidation) L1AtSafeHead(ctx context.Context, target eth.BlockID) (eth.BlockID, error) {
 	return eth.BlockID{}, nil
 }
-func (m *mockVNForInvalidation) FirstSafeHeadEntry(ctx context.Context) (eth.BlockID, eth.BlockID, error) {
-	return eth.BlockID{}, eth.BlockID{}, nil
+func (m *mockVNForInvalidation) FirstProvableSafeHeadNumber(ctx context.Context) (uint64, error) {
+	return 0, nil
 }
 func (m *mockVNForInvalidation) SyncStatus(ctx context.Context) (*eth.SyncStatus, error) {
 	return &eth.SyncStatus{}, nil
 }
+func (m *mockVNForInvalidation) IsEngineInitialELSyncing() bool { return false }
 
 var _ virtual_node.VirtualNode = (*mockVNForInvalidation)(nil)