feat(otel): add cold start metrics

cmd/ateapi/internal/controlapi/metrics.go

@@ -175,10 +175,12 @@ func (i *Instruments) recordLifecycleOp(ctx context.Context, op string, start ti
 }
 
 // lifecycleOpAttrs builds the resume/suspend/pause dimensions from workflow
-// state. Nil-safe, and omits the pool and snapshot-kind labels until they are
-// known so a failure before the assign/restore steps never emits an empty-string
-// series. snapshotKind is empty for suspend/pause, which do not restore.
-func lifecycleOpAttrs(actor *ateapipb.Actor, template *atev1alpha1.ActorTemplate, snapshotKind string) []attribute.KeyValue {
+// state. Nil-safe, and omits the pool, snapshot-kind and snapshot-scope labels
+// until they are known so a failure before the assign/restore steps never emits
+// an empty-string series. snapshotKind is empty for suspend/pause, which do not
+// restore; snapshotScope applies to all three and is what separates a restore
+// combined with the template's golden state from a plain one of the same kind.
+func lifecycleOpAttrs(actor *ateapipb.Actor, template *atev1alpha1.ActorTemplate, snapshotKind, snapshotScope string) []attribute.KeyValue {
 	attrs := []attribute.KeyValue{
 		ateattr.TemplateNameKey.String(actor.GetActorTemplateName()),
 		ateattr.TemplateNamespaceKey.String(actor.GetActorTemplateNamespace()),
@@ -192,6 +194,9 @@ func lifecycleOpAttrs(actor *ateapipb.Actor, template *atev1alpha1.ActorTemplate
 	if snapshotKind != "" {
 		attrs = append(attrs, ateattr.SnapshotKindKey.String(snapshotKind))
 	}
+	if snapshotScope != "" {
+		attrs = append(attrs, ateattr.SnapshotScopeKey.String(snapshotScope))
+	}
 	return attrs
 }
 

cmd/ateapi/internal/controlapi/metrics_test.go

@@ -216,7 +216,7 @@ func TestLifecycleOpDurationShape(t *testing.T) {
 		Spec: atev1alpha1.ActorTemplateSpec{SandboxClass: atev1alpha1.SandboxClassGvisor},
 	}
 	inst.recordLifecycleOp(context.Background(), ateattr.OperationResume, time.Now(), nil,
-		lifecycleOpAttrs(actor, template, ateattr.SnapshotKindLatest)...)
+		lifecycleOpAttrs(actor, template, ateattr.SnapshotKindLatest, ateattr.SnapshotScopeDataOnGolden)...)
 
 	dp := singleHistogramDP(t, reader, lifecycleOpDurationMetric)
 	assertAttrKeys(t, dp,
@@ -226,10 +226,31 @@ func TestLifecycleOpDurationShape(t *testing.T) {
 		ateattr.WorkerPoolNameKey,
 		ateattr.SandboxClassKey,
 		ateattr.SnapshotKindKey,
+		ateattr.SnapshotScopeKey,
 	)
 	if op, _ := attrString(dp, ateattr.ActorOperationNameKey); op != ateattr.OperationResume {
 		t.Errorf("operation = %q, want %q", op, ateattr.OperationResume)
 	}
+	// Kind and scope are independent: a data_on_golden restore of the actor's
+	// own latest snapshot must stay distinguishable from one of a local snapshot.
+	if scope, _ := attrString(dp, ateattr.SnapshotScopeKey); scope != ateattr.SnapshotScopeDataOnGolden {
+		t.Errorf("snapshot scope = %q, want %q", scope, ateattr.SnapshotScopeDataOnGolden)
+	}
+	if kind, _ := attrString(dp, ateattr.SnapshotKindKey); kind != ateattr.SnapshotKindLatest {
+		t.Errorf("snapshot kind = %q, want %q", kind, ateattr.SnapshotKindLatest)
+	}
+}
+
+// TestLifecycleOpAttrsOmitsUnknownScope guards the failure path: a resume that
+// dies before the restore request is built has no scope, and an empty-string
+// series would be indistinguishable from a real one.
+func TestLifecycleOpAttrsOmitsUnknownScope(t *testing.T) {
+	actor := &ateapipb.Actor{ActorTemplateName: "support-agent", ActorTemplateNamespace: "ate-agents"}
+	for _, kv := range lifecycleOpAttrs(actor, nil, "", "") {
+		if kv.Key == ateattr.SnapshotScopeKey || kv.Key == ateattr.SnapshotKindKey {
+			t.Errorf("attribute %s must be omitted while unknown, got %q", kv.Key, kv.Value.AsString())
+		}
+	}
 }
 
 // TestRecordLifecycleOp_OutcomeClassification asserts success omits error.type and

cmd/ateapi/internal/controlapi/workflow.go

@@ -187,7 +187,7 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, actorRef resources.Acto
 			return
 		}
 		w.instruments.recordLifecycleOp(ctx, ateattr.OperationResume, start, err,
-			lifecycleOpAttrs(state.Actor, state.ActorTemplate, state.SnapshotKind)...)
+			lifecycleOpAttrs(state.Actor, state.ActorTemplate, state.SnapshotKind, state.WireSnapshotScope)...)
 	}()
 
 	lockCtx, lock, err := w.acquireActorLock(ctx, actorRef)
@@ -222,7 +222,7 @@ func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.Act
 
 	defer func() {
 		w.instruments.recordLifecycleOp(ctx, ateattr.OperationSuspend, start, err,
-			lifecycleOpAttrs(state.Actor, state.ActorTemplate, "")...)
+			lifecycleOpAttrs(state.Actor, state.ActorTemplate, "", state.WireSnapshotScope)...)
 	}()
 
 	lockCtx, lock, err := w.acquireActorLock(ctx, actorRef)
@@ -256,7 +256,7 @@ func (w *ActorWorkflow) PauseActor(ctx context.Context, actorRef resources.Actor
 
 	defer func() {
 		w.instruments.recordLifecycleOp(ctx, ateattr.OperationPause, start, err,
-			lifecycleOpAttrs(state.Actor, state.ActorTemplate, "")...)
+			lifecycleOpAttrs(state.Actor, state.ActorTemplate, "", state.WireSnapshotScope)...)
 	}()
 
 	lockCtx, lock, err := w.acquireActorLock(ctx, actorRef)

cmd/ateapi/internal/controlapi/workflow_pause.go

@@ -41,8 +41,9 @@ type PauseInput struct {
 
 // PauseState holds the mutable state loaded and modified during execution.
 type PauseState struct {
-	Actor         *ateapipb.Actor
-	ActorTemplate *atev1alpha1.ActorTemplate
+	Actor             *ateapipb.Actor
+	ActorTemplate     *atev1alpha1.ActorTemplate
+	WireSnapshotScope string
 }
 
 type LoadActorForPauseStep struct {
@@ -168,6 +169,7 @@ func (s *CallAteletPauseStep) Execute(ctx context.Context, input *PauseInput, st
 		Scope:    toAteletSnapshotScope(state.ActorTemplate.Spec.SnapshotsConfig.OnPause),
 		ActorUid: state.Actor.GetMetadata().Uid,
 	}
+	state.WireSnapshotScope = ateattr.SnapshotScopeValue(req.Scope)
 
 	_, err = client.Checkpoint(ctx, req)
 	return maybeCrashActor(ctx, s.store, input.ActorRef, err, "while checkpointing workload", ateattr.OperationPause)

cmd/ateapi/internal/controlapi/workflow_resume.go

@@ -54,6 +54,9 @@ type ResumeState struct {
 	SnapshotLocation string
 	SnapshotScope    ateapipb.SnapshotContentScope
 	SnapshotKind     string
+	// WireSnapshotScope labels the restore requested, not the stored snapshot's
+	// SnapshotScope: a data snapshot restored on golden goes out as data_on_golden.
+	WireSnapshotScope string
 	// GoldenSnapshotLocation is the storage location of the ActorTemplate's
 	// golden snapshot. Populated only when the template's onResume
 	// configuration selects the golden snapshot as the boot source for the
@@ -581,6 +584,7 @@ func (s *CallAteletRestoreStep) Execute(ctx context.Context, input *ResumeInput,
 			req.Scope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN
 			req.GoldenSnapshotUriPrefix = state.GoldenSnapshotLocation
 		}
+		state.WireSnapshotScope = ateattr.SnapshotScopeValue(req.Scope)
 
 		_, err = client.Restore(ctx, req)
 		return maybeCrashActor(ctx, s.store, input.ActorRef, err, "while restoring workload", ateattr.OperationResume)
@@ -600,6 +604,7 @@ func (s *CallAteletRestoreStep) Execute(ctx context.Context, input *ResumeInput,
 		if state.GoldenSnapshotLocation != "" {
 			scope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN
 		}
+		state.WireSnapshotScope = ateattr.SnapshotScopeValue(scope)
 		req := &ateletpb.RestoreRequest{
 			TargetAteomUid:         assignment.GetWorkerPodUid(),
 			Atespace:               state.Actor.GetMetadata().GetAtespace(),

cmd/ateapi/internal/controlapi/workflow_suspend.go

@@ -42,9 +42,10 @@ type SuspendInput struct {
 
 // SuspendState holds the mutable state loaded and modified during execution.
 type SuspendState struct {
-	Actor         *ateapipb.Actor
-	ActorTemplate *atev1alpha1.ActorTemplate
-	SourceVersion int64
+	Actor             *ateapipb.Actor
+	ActorTemplate     *atev1alpha1.ActorTemplate
+	SourceVersion     int64
+	WireSnapshotScope string
 }
 
 type LoadActorForSuspendStep struct {
@@ -186,6 +187,7 @@ func (s *CallAteletSuspendStep) Execute(ctx context.Context, input *SuspendInput
 		Scope:    toAteletSnapshotScope(commitSnapshotScope(state.Actor.GetMetadata().GetAtespace(), state.ActorTemplate)),
 		ActorUid: state.Actor.GetMetadata().Uid,
 	}
+	state.WireSnapshotScope = ateattr.SnapshotScopeValue(req.Scope)
 
 	_, err = client.Checkpoint(ctx, req)
 	return maybeCrashActor(ctx, s.store, input.ActorRef, err, "while checkpointing workload", ateattr.OperationSuspend)