diff --git a/cmd/ateapi/internal/controlapi/actor_sizing_test.go b/cmd/ateapi/internal/controlapi/actor_sizing_test.go new file mode 100644 index 000000000..0b8791e65 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/actor_sizing_test.go @@ -0,0 +1,139 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "testing" + + atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// TestActorResourceLimits covers the actor-side extraction: the CPU/memory limits +// an ActorTemplate declares become the sandbox size and the scheduling floor. +func TestActorResourceLimits(t *testing.T) { + tests := []struct { + name string + res *corev1.ResourceRequirements + wantCPU int64 + wantMemory int64 + }{ + { + name: "nil resources yields zero", + res: nil, + wantCPU: 0, + wantMemory: 0, + }, + { + name: "cpu and memory limits are read", + res: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("4Gi"), + }, + }, + wantCPU: 2000, + wantMemory: 4 << 30, + }, + { + name: "millicpu is preserved", + res: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1500m")}, + }, + wantCPU: 1500, + wantMemory: 0, + }, + { + name: "requests are ignored; only limits size the actor", + res: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + wantCPU: 0, + wantMemory: 0, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tmpl := &atev1alpha1.ActorTemplate{Spec: atev1alpha1.ActorTemplateSpec{Resources: tc.res}} + cpu, mem := actorResourceLimits(tmpl) + if cpu != tc.wantCPU || mem != tc.wantMemory { + t.Fatalf("actorResourceLimits() = (%d, %d), want (%d, %d)", cpu, mem, tc.wantCPU, tc.wantMemory) + } + }) + } +} + +// TestWorkerCapacity covers the worker-side extraction: capacity comes from the +// ateom container's limits, not the pod total, and other containers are ignored. +func TestWorkerCapacity(t *testing.T) { + pod := func(ctrs ...corev1.Container) *corev1.Pod { + return &corev1.Pod{Spec: corev1.PodSpec{Containers: ctrs}} + } + limited := func(name, cpu, mem string) corev1.Container { + lim := corev1.ResourceList{} + if cpu != "" { + lim[corev1.ResourceCPU] = resource.MustParse(cpu) + } + if mem != "" { + lim[corev1.ResourceMemory] = resource.MustParse(mem) + } + return corev1.Container{Name: name, Resources: corev1.ResourceRequirements{Limits: lim}} + } + + tests := []struct { + name string + pod *corev1.Pod + wantCPU int64 + wantMemory int64 + }{ + { + name: "no ateom container yields zero", + pod: pod(limited("sidecar", "1", "1Gi")), + wantCPU: 0, + wantMemory: 0, + }, + { + name: "ateom container limits become capacity", + pod: pod(limited(ateomContainerName, "4", "8Gi")), + wantCPU: 4000, + wantMemory: 8 << 30, + }, + { + name: "only the ateom container counts, not the pod total", + pod: pod(limited("sidecar", "16", "64Gi"), limited(ateomContainerName, "2", "2Gi")), + wantCPU: 2000, + wantMemory: 2 << 30, + }, + { + name: "unset dimension reports zero", + pod: pod(limited(ateomContainerName, "2", "")), + wantCPU: 2000, + wantMemory: 0, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := workerCapacity(tc.pod) + if got.GetCpuMilli() != tc.wantCPU || got.GetMemoryBytes() != tc.wantMemory { + t.Fatalf("workerCapacity() = (%d, %d), want (%d, %d)", + got.GetCpuMilli(), got.GetMemoryBytes(), tc.wantCPU, tc.wantMemory) + } + }) + } +} diff --git a/cmd/ateapi/internal/controlapi/syncer.go b/cmd/ateapi/internal/controlapi/syncer.go index b3f56e364..93f3d0507 100644 --- a/cmd/ateapi/internal/controlapi/syncer.go +++ b/cmd/ateapi/internal/controlapi/syncer.go @@ -216,6 +216,7 @@ func (s *WorkerPoolSyncer) createOrUpdateWorker(ctx context.Context, key workerK SandboxClass: string(pool.Spec.SandboxClass), Labels: pool.GetLabels(), State: ateapipb.Worker_STATE_ACTIVE, + Capacity: workerCapacity(pod), } // TODO(thockin): for now this is the only place Workers are // created. If/when this becomes a regular API, validation should @@ -273,6 +274,38 @@ func isWorkerEligible(pod *corev1.Pod) bool { return pod.Status.PodIP != "" } +// ateomContainerName is the name of the container in a worker pod that hosts the +// actor's sandbox; its resource limits bound what an actor placed here can use. +const ateomContainerName = "ateom" + +// workerCapacity returns the worker pod's capacity for hosting an actor — CPU +// in millicores and memory in bytes — taken from the ateom container's resource +// limits. A dimension the pod does not limit reports 0, which the scheduler +// treats as "unknown" (unconstrained); a pod that limits neither reports nil +// rather than an all-zero message that says the same thing. The actor sandbox +// runs nested in the ateom container's cgroup, so that container's limits — not +// the pod total — are the relevant envelope. +func workerCapacity(pod *corev1.Pod) *ateapipb.WorkerCapacity { + var capacity ateapipb.WorkerCapacity + for i := range pod.Spec.Containers { + c := &pod.Spec.Containers[i] + if c.Name != ateomContainerName { + continue + } + if v := c.Resources.Limits.Cpu(); v != nil { + capacity.CpuMilli = v.MilliValue() + } + if v := c.Resources.Limits.Memory(); v != nil { + capacity.MemoryBytes = v.Value() + } + break + } + if capacity.CpuMilli == 0 && capacity.MemoryBytes == 0 { + return nil + } + return &capacity +} + // markWorkerDraining transitions a worker to STATE_DRAINING so the scheduler // stops routing new actors to it while its pod is Terminating. If the worker is // already gone or already draining there is nothing more to do — the Pod diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index 6666168d0..bdf999a3f 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -546,11 +546,32 @@ func workerAssignmentFrom(w *ateapipb.Worker) *ateapipb.WorkerAssignment { } } +// actorResourceLimits returns the actor's declared CPU (millicores) and memory +// (bytes) limits from its ActorTemplate, or 0 for a dimension the template did +// not set. These size the sandbox (supplied over the actor RPCs) and gate +// scheduling (a worker must have >= capacity). +func actorResourceLimits(tmpl *atev1alpha1.ActorTemplate) (cpuMilli, memBytes int64) { + res := tmpl.Spec.Resources + if res == nil { + return 0, 0 + } + if c := res.Limits.Cpu(); c != nil { + cpuMilli = c.MilliValue() + } + if m := res.Limits.Memory(); m != nil { + memBytes = m.Value() + } + return cpuMilli, memBytes +} + func schedulingConstraints(actor *ateapipb.Actor, tmpl *atev1alpha1.ActorTemplate) (scheduling.Constraints, error) { + cpuMilli, memBytes := actorResourceLimits(tmpl) c := scheduling.Constraints{ SandboxClass: string(tmpl.Spec.SandboxClass), ActorSelector: labels.SelectorFromSet(labels.Set(actor.GetWorkerSelector().GetMatchLabels())), RequiredNodes: actor.GetLocalSnapshotInfo().GetNodeVmsWithLocalSnapshots(), + CPUMilli: cpuMilli, + MemoryBytes: memBytes, } if tmpl.Spec.WorkerSelector != nil { sel, err := metav1.LabelSelectorAsSelector(tmpl.Spec.WorkerSelector) @@ -613,6 +634,10 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou } egressGateway := w.egressGateway() + // The actor's declared limits ride the RPC down to the sandbox so it is sized + // to the actor (replacing the worker-pod downward-API approach). + cpuMilli, memBytes := actorResourceLimits(actorTemplate) + if local := actor.GetLocalSnapshotInfo(); local != nil { slog.InfoContext(ctx, "Actor has snapshot; Restoring from snapshot") tele.SnapshotKind = ateattr.SnapshotKindLocal @@ -626,6 +651,8 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou Spec: workloadSpec, ActorUid: actor.GetMetadata().Uid, EgressGateway: egressGateway, + CpuMilli: cpuMilli, + MemoryBytes: memBytes, } req.Type = ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL req.Config = &ateletpb.RestoreRequest_LocalConfig{ @@ -681,6 +708,8 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou GoldenSnapshotUri: src.GoldenSnapshotURI.String(), ActorUid: actor.GetMetadata().Uid, EgressGateway: egressGateway, + CpuMilli: cpuMilli, + MemoryBytes: memBytes, } _, err = client.Restore(ctx, req) return tele, maybeCrashActor(ctx, w.store, actorRef, err, "while restoring durable snapshot", ateattr.OperationResume) @@ -706,6 +735,8 @@ func (w *ActorWorkflow) ensureAteletRestored(ctx context.Context, actorRef resou Spec: workloadSpec, ActorUid: actor.GetMetadata().Uid, EgressGateway: egressGateway, + CpuMilli: cpuMilli, + MemoryBytes: memBytes, } _, err = client.Run(ctx, req) return tele, maybeCrashActor(ctx, w.store, actorRef, err, "while creating workload from spec", ateattr.OperationResume) diff --git a/cmd/ateapi/internal/scheduling/scheduling.go b/cmd/ateapi/internal/scheduling/scheduling.go index 0c69d6b15..cf293a464 100644 --- a/cmd/ateapi/internal/scheduling/scheduling.go +++ b/cmd/ateapi/internal/scheduling/scheduling.go @@ -41,6 +41,15 @@ type Constraints struct { // on one of these nodes. Used when the actor's latest snapshot is local // to specific node VMs. RequiredNodes []string + + // CPUMilli and MemoryBytes are the actor's declared resource limits, from + // the ActorTemplate. A worker is eligible only if its reported capacity is + // >= these. Zero means "unconstrained" for that dimension (the actor did not + // declare a limit), and a worker that reports zero capacity for a dimension + // is treated as unconstrained too, so placement is never blocked by missing + // data (matching the pre-capacity behavior). + CPUMilli int64 + MemoryBytes int64 } // ErrNoCapacity is returned by Schedule when no free worker satisfies the @@ -136,5 +145,17 @@ func (s *scheduler) Applies(worker *ateapipb.Worker, constraints Constraints) bo return false } + // The worker must be able to contain the actor's declared limits. A zero + // constraint (actor declared no limit) or zero worker capacity (capacity + // unknown) is treated as unconstrained, so placement is never blocked by + // missing data. + capacity := worker.GetCapacity() + if constraints.CPUMilli > 0 && capacity.GetCpuMilli() > 0 && capacity.GetCpuMilli() < constraints.CPUMilli { + return false + } + if constraints.MemoryBytes > 0 && capacity.GetMemoryBytes() > 0 && capacity.GetMemoryBytes() < constraints.MemoryBytes { + return false + } + return len(constraints.RequiredNodes) == 0 || slices.Contains(constraints.RequiredNodes, worker.GetNodeName()) } diff --git a/cmd/ateapi/internal/scheduling/scheduling_test.go b/cmd/ateapi/internal/scheduling/scheduling_test.go index 55124bd31..6ed21befa 100644 --- a/cmd/ateapi/internal/scheduling/scheduling_test.go +++ b/cmd/ateapi/internal/scheduling/scheduling_test.go @@ -120,6 +120,47 @@ func TestSchedule(t *testing.T) { constraints: Constraints{SandboxClass: "gvisor"}, wantPod: "w-active", }, + { + name: "worker with too little cpu capacity is skipped", + fleet: fleet{ + worker("w-small", "gvisor", "node-a", tierTwo, withCapacity(1000, 8<<30)), + worker("w-big", "gvisor", "node-a", tierTwo, withCapacity(4000, 8<<30)), + }, + constraints: Constraints{SandboxClass: "gvisor", CPUMilli: 2000}, + wantPod: "w-big", + }, + { + name: "worker with too little memory capacity is skipped", + fleet: fleet{ + worker("w-small", "gvisor", "node-a", tierTwo, withCapacity(4000, 1<<30)), + worker("w-big", "gvisor", "node-a", tierTwo, withCapacity(4000, 4<<30)), + }, + constraints: Constraints{SandboxClass: "gvisor", MemoryBytes: 2 << 30}, + wantPod: "w-big", + }, + { + name: "no worker with enough capacity yields ErrNoCapacity", + fleet: fleet{ + worker("w-small", "gvisor", "node-a", tierTwo, withCapacity(1000, 1<<30)), + }, + constraints: Constraints{SandboxClass: "gvisor", CPUMilli: 2000}, + }, + { + name: "zero worker capacity is treated as unconstrained", + fleet: fleet{ + worker("w-unknown", "gvisor", "node-a", tierTwo), + }, + constraints: Constraints{SandboxClass: "gvisor", CPUMilli: 2000, MemoryBytes: 2 << 30}, + wantPod: "w-unknown", + }, + { + name: "zero constraint ignores worker capacity", + fleet: fleet{ + worker("w-tiny", "gvisor", "node-a", tierTwo, withCapacity(100, 1<<20)), + }, + constraints: Constraints{SandboxClass: "gvisor"}, + wantPod: "w-tiny", + }, { name: "empty fleet", fleet: fleet{}, @@ -269,6 +310,12 @@ func assigned(atespace, name string) func(*ateapipb.Worker) { } } +func withCapacity(cpuMilli, memBytes int64) func(*ateapipb.Worker) { + return func(w *ateapipb.Worker) { + w.Capacity = &ateapipb.WorkerCapacity{CpuMilli: cpuMilli, MemoryBytes: memBytes} + } +} + // firstIntn always picks the first candidate, making Schedule deterministic. func firstIntn(int) int { return 0 } diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go index 89b7ac273..9e73c6d73 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go @@ -753,11 +753,13 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf WithAdd(ateomGvisorCapabilities...)). WithAppArmorProfile(corev1ac.AppArmorProfile(). WithType(corev1.AppArmorProfileTypeUnconfined))). - WithEnv(corev1ac.EnvVar(). - WithName("POD_UID"). - WithValueFrom(corev1ac.EnvVarSource(). - WithFieldRef(corev1ac.ObjectFieldSelector(). - WithFieldPath("metadata.uid")))). + WithEnv( + corev1ac.EnvVar(). + WithName("POD_UID"). + WithValueFrom(corev1ac.EnvVarSource(). + WithFieldRef(corev1ac.ObjectFieldSelector(). + WithFieldPath("metadata.uid"))), + ). WithVolumeMounts( corev1ac.VolumeMount(). WithName("run-ateom"). diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 0fcd67c02..f6ced5b36 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -454,6 +454,8 @@ func (s *AteomHerder) Run(ctx context.Context, req *ateletpb.RunRequest) (resp * Spec: buildAteomWorkloadSpec(req.GetSpec()), ActorUid: actorUID, EgressGateway: toAteomEgressGateway(req.GetEgressGateway()), + CpuMilli: req.GetCpuMilli(), + MemoryBytes: req.GetMemoryBytes(), }); err != nil { return nil, fmt.Errorf("while calling ateom.RunWorkload: %w", err) } @@ -1091,6 +1093,8 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) Scope: toAteomSnapshotScope(req.GetScope()), ActorUid: req.GetActorUid(), EgressGateway: toAteomEgressGateway(req.GetEgressGateway()), + CpuMilli: req.GetCpuMilli(), + MemoryBytes: req.GetMemoryBytes(), // Informational: for DATA_ON_GOLDEN the golden snapshot's files are // already staged into the restore dir by the combined download above; // ateom restores from the shared dir and never fetches this URI. diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 415f494a8..af4f80661 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -47,6 +47,7 @@ import ( "github.com/agent-substrate/substrate/internal/readyz" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/internal/serverboot" + "github.com/agent-substrate/substrate/internal/sizing" "github.com/agent-substrate/substrate/internal/version" "github.com/hashicorp/go-reap" "github.com/spf13/pflag" @@ -597,6 +598,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload rcmd := &runsc{ path: req.GetRunscPath(), actorUID: req.GetActorUid(), + size: sizing.FromLimits(req.GetCpuMilli(), req.GetMemoryBytes()), } var containersToDelete []string defer func() { @@ -621,7 +623,6 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload } } }() - // Create and start pause container. The bundle rootfs is composed here — // an overlay of the node's cached image layers plus the bundle's private // upper — because mounting is ateom's job (atelet runs with no @@ -698,6 +699,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // * After we exit, atelet will upload checkpoint to GCS // * After we exit, atelet will tear down OCI bundles and reset the actor directory. + // Checkpoint only saves state; no sizing is applied, so size is left zero. rcmd := &runsc{ path: req.GetRunscPath(), actorUID: req.GetActorUid(), @@ -872,6 +874,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore rcmd := &runsc{ path: req.GetRunscPath(), actorUID: req.GetActorUid(), + size: sizing.FromLimits(req.GetCpuMilli(), req.GetMemoryBytes()), } var containersToDelete []string defer func() { @@ -893,7 +896,6 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } } }() - checkpointDir := ateompath.RestoreStateDir(req.GetActorUid()) // Compose the pause rootfs before create (see RunWorkload). runsc restore diff --git a/cmd/ateom-gvisor/runsc.go b/cmd/ateom-gvisor/runsc.go index 4c89b799f..0d517a1b4 100644 --- a/cmd/ateom-gvisor/runsc.go +++ b/cmd/ateom-gvisor/runsc.go @@ -29,11 +29,15 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/sizing" ) type runsc struct { path string actorUID string + // size is the actor's declared limits, supplied on the RunWorkload / + // RestoreWorkload RPC; ensureContainerCgroupsPath writes it into the OCI spec. + size sizing.SandboxSize } // nvproxyGlobalArgs returns the runsc global flags for GPU sandboxes, enabling @@ -74,10 +78,13 @@ func (r *runsc) ensureContainerCgroupsPath(containerName string) error { if spec.Linux == nil { spec.Linux = &specs.Linux{} } - if spec.Linux.CgroupsPath != "" { - return nil + if spec.Linux.CgroupsPath == "" { + spec.Linux.CgroupsPath = "/" + containerName } - spec.Linux.CgroupsPath = "/" + containerName + // Right-size the per-container cgroup leaf to the actor's declared limits; + // runsc applies spec.Linux.Resources when it creates the leaf. Shared with the + // micro-VM runtime via internal/sizing. + r.size.ApplyToOCISpec(&spec) out, err := json.MarshalIndent(&spec, "", " ") if err != nil { return fmt.Errorf("marshaling %q: %w", specPath, err) @@ -107,6 +114,10 @@ func (r *runsc) cmdCreate(ctx context.Context, out io.Writer, containerName stri // "-log-packets", // "-strace", "-root", ateompath.RunSCStateDir(r.actorUID), + // Provision the sentry's vCPU count from the cgroup CPU quota written by + // sizing.ApplyToOCISpec, so the sandbox is sized to the pod's limit (runsc + // otherwise sizes to all host CPUs). Global flag: before the subcommand. + "--cpu-num-from-quota", } args = append(args, nvproxyGlobalArgs()...) args = append(args, @@ -253,6 +264,8 @@ func (r *runsc) cmdRestore(ctx context.Context, out io.Writer, containerName, ch // "-log-packets", // "-strace", "-root", ateompath.RunSCStateDir(r.actorUID), + // Match cmdCreate: size the restored sentry from the cgroup CPU quota. + "--cpu-num-from-quota", } restoreArgs = append(restoreArgs, nvproxyGlobalArgs()...) restoreArgs = append(restoreArgs, diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index ccedb0c63..f98e98750 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -55,12 +55,13 @@ import ( ) var ( - podUID = flag.String("pod-uid", "", "The UID of the current pod") - chBinary = flag.String("cloud-hypervisor-binary", "cloud-hypervisor", "Path to the cloud-hypervisor binary (used to relaunch on restore).") - kataConfig = flag.String("kata-config", "", "Path to a kata configuration.toml (passed to the shim as KATA_CONF_FILE). Empty uses kata's default. atelet generates one pointing at runtime-fetched assets.") - kataDebug = flag.Bool("kata-debug", false, "Verbose kata-agent debugging: raise the guest agent log level and forward the guest console (incl. agent logs) into the pod logs.") - showVersion = flag.Bool("version", false, "Print version and exit.") - logLevelFlag = flag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") + podUID = flag.String("pod-uid", "", "The UID of the current pod") + chBinary = flag.String("cloud-hypervisor-binary", "cloud-hypervisor", "Path to the cloud-hypervisor binary (used to relaunch on restore).") + kataConfig = flag.String("kata-config", "", "Path to a kata configuration.toml (passed to the shim as KATA_CONF_FILE). Empty uses kata's default. atelet generates one pointing at runtime-fetched assets.") + kataDebug = flag.Bool("kata-debug", false, "Verbose kata-agent debugging: raise the guest agent log level and forward the guest console (incl. agent logs) into the pod logs.") + vmmMemReserve = flag.Int("vmm-mem-reserve-mib", vmmMemReserveMiB, "Guest RAM (MiB) held back from the pod's memory limit for the cloud-hypervisor VMM + virtiofsd, which run as host processes in the pod cgroup alongside the guest RAM. Prevents the pod OOMing when the VM is sized to the pod's memory limit.") + showVersion = flag.Bool("version", false, "Print version and exit.") + logLevelFlag = flag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") atunnelListenAddress = flag.String("atunnel-listen-address", "0.0.0.0:443", "Address for actor ingress HTTPS") workerCredentialBundle = flag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Worker Pod credential bundle used by atunnel for inbound serving and outbound mTLS") @@ -214,7 +215,7 @@ func do(ctx context.Context) error { grpc.StatsHandler(otelgrpc.NewServerHandler()), grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor), ) - ateompb.RegisterAteomServer(svr, NewService(*podUID, *chBinary, *kataConfig, *kataDebug, interiorNetNS, actorLogger, atunnelIngress, atunnelEgress, atunnelEgressPort, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle)) + ateompb.RegisterAteomServer(svr, NewService(*podUID, *chBinary, *kataConfig, *kataDebug, *vmmMemReserve, interiorNetNS, actorLogger, atunnelIngress, atunnelEgress, atunnelEgressPort, *workerCredentialBundle, *podIdentityTrustBundle, *egressGatewayTrustBundle)) reflection.Register(svr) slog.InfoContext(ctx, "ateom-microvm serving", slog.String("socket", sockPath)) @@ -265,6 +266,11 @@ type AteomService struct { kataConfig string kataDebug bool + // memReserveMiB is guest RAM (MiB) held back from the pod's memory limit for + // the cloud-hypervisor VMM + virtiofsd (host processes sharing the pod cgroup + // with the guest RAM). Set from --vmm-mem-reserve-mib. + memReserveMiB int + // interiorNetNS hosts the per-activation actor veth peer (see net.go); // kata is pointed at it. interiorNetNS netns.NsHandle @@ -339,12 +345,13 @@ type AteomService struct { var _ ateompb.AteomServer = (*AteomService)(nil) // NewService creates a new AteomService. -func NewService(podUID, chBinary, kataConfig string, kataDebug bool, interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelIngress *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath string) *AteomService { +func NewService(podUID, chBinary, kataConfig string, kataDebug bool, memReserveMiB int, interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelIngress *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, workerCredentialBundlePath, podIdentityTrustBundlePath, egressGatewayTrustBundlePath string) *AteomService { return &AteomService{ podUID: podUID, chBinary: chBinary, kataConfig: kataConfig, kataDebug: kataDebug, + memReserveMiB: memReserveMiB, interiorNetNS: interiorNetNS, actorLogger: actorLogger, atunnelIngress: atunnelIngress, diff --git a/cmd/ateom-microvm/restore.go b/cmd/ateom-microvm/restore.go index 229640e58..96b4dfb31 100644 --- a/cmd/ateom-microvm/restore.go +++ b/cmd/ateom-microvm/restore.go @@ -36,6 +36,7 @@ import ( "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/readyz" "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/internal/sizing" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -87,14 +88,14 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } p := actorBootParams{ - actorRef: resources.ActorRef{Atespace: req.GetAtespace(), Name: req.GetActorName()}, - actorUID: req.GetActorUid(), - templateNS: req.GetActorTemplateNamespace(), - templateName: req.GetActorTemplateName(), - containers: req.GetSpec().GetContainers(), - assetPaths: req.GetRuntimeAssetPaths(), - + actorRef: resources.ActorRef{Atespace: req.GetAtespace(), Name: req.GetActorName()}, + actorUID: req.GetActorUid(), + templateNS: req.GetActorTemplateNamespace(), + templateName: req.GetActorTemplateName(), + containers: req.GetSpec().GetContainers(), + assetPaths: req.GetRuntimeAssetPaths(), egressGateway: req.GetEgressGateway(), + size: sizing.FromLimits(req.GetCpuMilli(), req.GetMemoryBytes()), } restoreDir := ateompath.RestoreStateDir(p.actorUID) durableDir := ateompath.DurableDirVolumeMountsDir(p.actorUID) @@ -204,7 +205,15 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, if len(containers) > maxActorContainers { return status.Errorf(codes.Unimplemented, "ateom-microvm supports at most %d containers, got %d", maxActorContainers, len(containers)) } - ctrs, err := s.buildActorContainers(actorUID, containers) + // The VM's RAM comes from the snapshot, so a limit the current VMM reserve can + // no longer satisfy (e.g. --vmm-mem-reserve-mib was raised after the snapshot + // was taken) has to fail here rather than silently pair the guest with a cgroup + // limit larger than its RAM. + guestSize, err := s.guestSize(p.size) + if err != nil { + return err + } + ctrs, err := s.buildActorContainers(actorUID, containers, guestSize) if err != nil { return err } diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index 2b3dc406b..c2c5ff403 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -39,6 +39,7 @@ import ( "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/readyz" "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/internal/sizing" specs "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/sys/unix" "google.golang.org/grpc/codes" @@ -116,6 +117,21 @@ const ( assetVirtiofsd = "virtiofsd" ) +// vmmMemReserveMiB is the DEFAULT guest RAM held back from the pod's memory limit +// for the cloud-hypervisor VMM + virtiofsd, which run as host processes in the same +// pod cgroup as the guest RAM; without a margin the pod OOMs. Overridable per +// deployment via --vmm-mem-reserve-mib (see AteomService.memReserveMiB). +const vmmMemReserveMiB = 256 + +// minGuestMemMiB is the floor for guest RAM (the declared limit minus the VMM +// reserve); a declared memory limit that leaves less is rejected at cold boot with a +// clear error instead of being silently honored (see resolveGuestMemMiB), since too +// little RAM makes the guest hang on boot rather than fail cleanly. It is a +// conservative estimate; calibrate against a measured kata boot minimum if a tighter +// bound is needed, and keep the admission floor on ActorTemplate.spec.resources in +// sync (it is this value + vmmMemReserveMiB). +const minGuestMemMiB = 256 + // maxActorContainers is a sanity cap on containers per actor (all share the one // micro-VM + virtiofsd). 25 is far above any real pod. const maxActorContainers = 25 @@ -216,14 +232,14 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload } p := actorBootParams{ - actorRef: resources.ActorRef{Atespace: req.GetAtespace(), Name: req.GetActorName()}, - actorUID: req.GetActorUid(), - templateNS: req.GetActorTemplateNamespace(), - templateName: req.GetActorTemplateName(), - containers: req.GetSpec().GetContainers(), - assetPaths: req.GetRuntimeAssetPaths(), - + actorRef: resources.ActorRef{Atespace: req.GetAtespace(), Name: req.GetActorName()}, + actorUID: req.GetActorUid(), + templateNS: req.GetActorTemplateNamespace(), + templateName: req.GetActorTemplateName(), + containers: req.GetSpec().GetContainers(), + assetPaths: req.GetRuntimeAssetPaths(), egressGateway: req.GetEgressGateway(), + size: sizing.FromLimits(req.GetCpuMilli(), req.GetMemoryBytes()), } s.actorLogger.EmitLifecycleLog("Actor starting", p.actorRef, p.actorUID, p.templateNS, p.templateName) @@ -262,6 +278,10 @@ type actorBootParams struct { assetPaths map[string]string // egressGateway is nil unless actor TCP should be redirected through atunnel. egressGateway *ateompb.EgressGateway + // size is the actor's declared limits (from the ActorTemplate), supplied on + // the RunWorkload / RestoreWorkload RPC. It sizes the VM (vCPUs, memory) and + // the guest container cgroup. Zero fields keep the kata defaults. + size sizing.SandboxSize } // actorAttribution regroups the actor fields that arrived on the Run/Restore @@ -363,15 +383,38 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re } }() + // Guest sizing + agent kernel params from the kata config. + memMiB, vcpus, kparams, err := s.guestConfig(rr) + if err != nil { + return err + } + + // Right-size the VM to the actor's declared limits (see internal/sizing), + // keeping the kata-config values above as the fallback when a limit is unset. + // vCPUs round up; VM RAM reserves a fixed margin for the VMM + virtiofsd, which + // share the pod cgroup with the guest RAM. A declared memory limit the reserve + // leaves too small to boot is rejected (resolveGuestMemMiB) rather than silently + // falling back to the larger kata default. NB: a FULL-scope snapshot restore + // reuses the size baked into the snapshot (restoreFullScope), so resizing an + // existing actor takes effect on its next cold boot. + sz := p.size + if v := sz.VCPUs(); v > 0 { + vcpus = v + } + memMiB, err = resolveGuestMemMiB(sz.MemoryBytes, s.memReserveMiB, memMiB) + if err != nil { + return err + } + // Prepare each container's OCI spec + record its bundle rootfs (the overlay RO // lower). No host disk — the rootfs is overlay(virtio-fs lower + guest-tmpfs upper). - ctrs, err := s.buildActorContainers(actorUID, containers) + // Size the guest container cgroup to the post-reserve guest RAM (matching memMiB) + // so the in-guest cgroup limit binds against actual guest memory. + guestSize, err := s.guestSize(sz) if err != nil { return err } - - // Guest sizing + agent kernel params from the kata config. - memMiB, vcpus, kparams, err := s.guestConfig(rr) + ctrs, err := s.buildActorContainers(actorUID, containers, guestSize) if err != nil { return err } @@ -531,13 +574,13 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re // built — the rootfs is overlay(virtio-fs RO lower + guest-tmpfs upper); the lowers // are bound into virtiofsd's shared dir in stageOverlayLowers after the sandbox state // is clean. Both RunWorkload and RestoreWorkload go through here. -func (s *AteomService) buildActorContainers(actorUID string, containers []*ateompb.Container) ([]actorContainer, error) { +func (s *AteomService) buildActorContainers(actorUID string, containers []*ateompb.Container, size sizing.SandboxSize) ([]actorContainer, error) { netnsPath := ateompath.AteomNetNSPath(s.podUID) ctrs := make([]actorContainer, len(containers)) for i, c := range containers { cn := c.GetName() bundle := ateompath.OCIBundlePath(actorUID, cn) - spec, err := ensureKataCompatibleSpec(bundle, actorUID, netnsPath) + spec, err := ensureKataCompatibleSpec(bundle, actorUID, netnsPath, size) if err != nil { return nil, fmt.Errorf("while preparing kata OCI spec for %q: %w", cn, err) } @@ -612,6 +655,68 @@ func (s *AteomService) guestConfig(rr resolvedRuntime) (memMiB, vcpus int, kpara return cfg.MemoryMiB, cfg.VCPUs, kparams, nil } +// resolveGuestMemMiB returns the micro-VM guest RAM (MiB) for an actor's declared +// memory limit. declaredBytes == 0 means "unset" and returns fallbackMiB (the +// kata-config default). Otherwise the guest gets the declared memory minus the VMM +// reserve; if that leaves less than a bootable minimum it errors — naming the limit, +// the reserve, and the minimum — instead of silently reverting to the (larger) +// fallback, which would boot the actor bigger than the worker was sized for and OOM +// the pod (see vmmMemReserveMiB, minGuestMemMiB, and internal/sizing). +func resolveGuestMemMiB(declaredBytes int64, reserveMiB, fallbackMiB int) (int, error) { + if declaredBytes <= 0 { + return fallbackMiB, nil + } + declaredMiB := int(declaredBytes / (1024 * 1024)) + m := declaredMiB - reserveMiB + if m < minGuestMemMiB { + return 0, fmt.Errorf("actor memory limit %dMiB is too small for a micro-VM: "+ + "the %dMiB VMM reserve leaves %dMiB, below the %dMiB guest minimum", + declaredMiB, reserveMiB, m, minGuestMemMiB) + } + return m, nil +} + +// guestSize translates an actor's declared limits into the effective in-guest +// limits: CPU passes through unchanged (kata-agent sets the CFS quota), while +// memory is reduced by the VMM reserve so the container cgroup limit inside the +// guest matches the guest VM's actual RAM rather than the (larger) outer limit. +// An unset (zero) memory limit passes through unset. +// +// The VM's RAM is the real envelope; the per-container cgroup is belt-and-braces. +// Every container in a multi-container actor gets the same limit — the whole +// guest RAM — so containers are not bounded relative to each other, and because +// the guest kernel, agent and init consume part of that RAM the workload reaches +// the guest OOM killer just before the cgroup limit binds. Sizing the cgroup to +// the guest's RAM keeps the two numbers from contradicting each other; enforcing +// a per-container share would need those overheads subtracted first. +// +// CPU has no equivalent of the VMM reserve, so "unchanged" above is about the +// number, not about what the workload gets. On gVisor the sentry is the workload +// and shares the sandbox's cgroup leaf, so the declared limit covers everything +// the sandbox costs the host. Here the limit reaches the guest cgroup intact, but +// cloud-hypervisor's vCPU threads, virtiofsd and ateom are host processes drawing +// on the same worker-pod CPU quota, and the scheduler's capacity check (>=) sets +// none of it aside — so the workload runs on somewhat less than it declared, and +// the in-guest quota is throttled by the host before it ever binds. The asymmetry +// with memory is deliberate: an unreduced memory limit would push the pod past its +// own and get it OOM-killed, whereas CPU is compressible, so the shortfall only +// slows the workload down. Carving out a CPU reserve is left for a follow-up. +// +// An error means the declared limit cannot be honored (see resolveGuestMemMiB); +// callers must not fall back to the unreduced size, which is the mismatch this +// translation exists to avoid. +func (s *AteomService) guestSize(sz sizing.SandboxSize) (sizing.SandboxSize, error) { + if sz.MemoryBytes <= 0 { + return sz, nil + } + memMiB, err := resolveGuestMemMiB(sz.MemoryBytes, s.memReserveMiB, 0) + if err != nil { + return sz, err + } + sz.MemoryBytes = int64(memMiB) * 1024 * 1024 + return sz, nil +} + // buildVMConfig assembles the cloud-hypervisor VmConfig. The kernel cmdline replicates // kata's clh boot cmdline; beyond the base params it must set // systemd.unit=kata-containers.target (else the guest powers off ~6s in) and mask diff --git a/cmd/ateom-microvm/run_test.go b/cmd/ateom-microvm/run_test.go index 97338f9fd..fca582b1e 100644 --- a/cmd/ateom-microvm/run_test.go +++ b/cmd/ateom-microvm/run_test.go @@ -23,6 +23,8 @@ import ( "path/filepath" "testing" "time" + + "github.com/agent-substrate/substrate/internal/sizing" ) // A vsock socket that has gone missing means cloud-hypervisor stopped the VM @@ -94,3 +96,81 @@ func TestDialAgentRetryContextCanceled(t *testing.T) { t.Errorf("dialAgentRetry(%q) error = %v, want context.Canceled", path, err) } } + +// resolveGuestMemMiB must honor a declared limit (minus the VMM reserve), fall back +// to the kata default only when the limit is unset, and error — never silently boot +// bigger than declared — when the reserve leaves too little to boot a guest. +func TestResolveGuestMemMiB(t *testing.T) { + const ( + mib = 1024 * 1024 + reserve = 256 // vmmMemReserveMiB + fallback = 2048 // kata-config default + ) + tests := []struct { + name string + declaredMiB int64 // 0 => unset + wantMiB int + wantErr bool + }{ + {name: "unset falls back to kata default", declaredMiB: 0, wantMiB: fallback}, + {name: "declared honored minus reserve", declaredMiB: 1536, wantMiB: 1536 - reserve}, + {name: "just above minimum", declaredMiB: reserve + minGuestMemMiB, wantMiB: minGuestMemMiB}, + {name: "reserve exactly swallows limit", declaredMiB: reserve, wantErr: true}, + {name: "limit below reserve", declaredMiB: 128, wantErr: true}, + {name: "boot-hang band (too little guest RAM)", declaredMiB: 320, wantErr: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := resolveGuestMemMiB(tc.declaredMiB*mib, reserve, fallback) + if tc.wantErr { + if err == nil { + t.Fatalf("resolveGuestMemMiB(%dMiB) = %d, nil; want an error", tc.declaredMiB, got) + } + return + } + if err != nil { + t.Fatalf("resolveGuestMemMiB(%dMiB) unexpected error: %v", tc.declaredMiB, err) + } + if got != tc.wantMiB { + t.Errorf("resolveGuestMemMiB(%dMiB) = %d, want %d", tc.declaredMiB, got, tc.wantMiB) + } + }) + } +} + +// guestSize translates an actor's declared limits to effective in-guest limits: +// CPU is preserved while memory is reduced by the VMM reserve. +func TestGuestSize(t *testing.T) { + const ( + mib = 1024 * 1024 + reserve = 256 + ) + s := &AteomService{memReserveMiB: reserve} + + // Declared memory limit is reduced by reserve. + got, err := s.guestSize(sizing.SandboxSize{MilliCPU: 2000, MemoryBytes: 1024 * mib}) + want := sizing.SandboxSize{MilliCPU: 2000, MemoryBytes: (1024 - reserve) * mib} + if err != nil { + t.Fatalf("guestSize(1024MiB) unexpected error: %v", err) + } + if got != want { + t.Errorf("guestSize(1024MiB) = %+v, want %+v", got, want) + } + + // Unset memory (0) is left unset. + gotUnset, err := s.guestSize(sizing.SandboxSize{MilliCPU: 1000, MemoryBytes: 0}) + wantUnset := sizing.SandboxSize{MilliCPU: 1000, MemoryBytes: 0} + if err != nil { + t.Fatalf("guestSize(unset) unexpected error: %v", err) + } + if gotUnset != wantUnset { + t.Errorf("guestSize(unset) = %+v, want %+v", gotUnset, wantUnset) + } + + // A limit the reserve leaves too small to boot errors rather than returning + // the unreduced size, which would leave the cgroup limit above the VM's RAM. + tooSmall := sizing.SandboxSize{MilliCPU: 1000, MemoryBytes: (reserve + 1) * mib} + if gotErr, err := s.guestSize(tooSmall); err == nil { + t.Errorf("guestSize(%dMiB) = %+v, nil; want an error", reserve+1, gotErr) + } +} diff --git a/cmd/ateom-microvm/spec.go b/cmd/ateom-microvm/spec.go index 7962bc5aa..3e328b486 100644 --- a/cmd/ateom-microvm/spec.go +++ b/cmd/ateom-microvm/spec.go @@ -24,6 +24,8 @@ import ( "strings" specs "github.com/opencontainers/runtime-spec/specs-go" + + "github.com/agent-substrate/substrate/internal/sizing" ) // ensureKataCompatibleSpec augments the bundle's config.json with the fields @@ -31,7 +33,7 @@ import ( // Without linux.resources, kata's ContainerConfig nil-derefs and the shim // crashes. This shaper is a bridge; a future atelet change should emit // runtime-appropriate specs so it can retire. -func ensureKataCompatibleSpec(bundle, id, netnsPath string) (*specs.Spec, error) { +func ensureKataCompatibleSpec(bundle, id, netnsPath string, size sizing.SandboxSize) (*specs.Spec, error) { specPath := filepath.Join(bundle, "config.json") b, err := os.ReadFile(specPath) if err != nil { @@ -51,6 +53,11 @@ func ensureKataCompatibleSpec(bundle, id, netnsPath string) (*specs.Spec, error) if spec.Linux.CgroupsPath == "" { spec.Linux.CgroupsPath = "/ateomchv/" + id } + // Right-size the guest container cgroup to the actor's declared limits; the + // kata-agent applies spec.Linux.Resources inside the VM. Shared with the gVisor + // runtime via internal/sizing; overlays the device allowlist + CPU shares set + // by defaultKataResources. + size.ApplyToOCISpec(&spec) // atelet's spec carries gVisor pause-model CRI annotations // (container-type=container, sandbox-id=pause). kata reads those and waits diff --git a/demos/counter/counter-microvm.yaml.tmpl b/demos/counter/counter-microvm.yaml.tmpl index 960bd9018..2b38135a4 100644 --- a/demos/counter/counter-microvm.yaml.tmpl +++ b/demos/counter/counter-microvm.yaml.tmpl @@ -38,6 +38,25 @@ spec: sandboxClass: microvm sandboxConfigName: microvm ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-microvm + template: + # Per-worker resources: these size the worker POD and advertise its + # scheduling capacity (the per-actor ceiling). The micro-VM sandbox itself is + # sized by the ActorTemplate's spec.resources below, not by these; keep the + # actor memory below this limit so the VMM reserve fits (see internal/sizing). + # + # Capacity comes from the limits (workerCapacity reads limits only), while the + # kube-scheduler packs nodes by the requests — so the CPU request is set well + # below the limit to keep the pool placeable on a small node (a single kind + # node in CI, shared with the gVisor demo pool) without changing what an actor + # is allowed to use. Memory is not compressible, so its request matches the + # limit: the guest RAM plus the VMM reserve has to be really there. + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 500m + memory: 2Gi --- @@ -64,6 +83,14 @@ spec: volumeMounts: - name: data mountPath: /home/counter + # Sandbox size: an actor occupies its whole worker. Size vCPUs + guest RAM at + # (or below) the pool's per-worker capacity above, leaving headroom under the + # worker memory limit for the VMM reserve. ateom applies these to the guest + # (see internal/sizing). + resources: + limits: + cpu: "2" + memory: 1536Mi workerSelector: matchLabels: workload: counter-microvm diff --git a/demos/counter/counter.yaml.tmpl b/demos/counter/counter.yaml.tmpl index 892a21f99..ab78b59b4 100644 --- a/demos/counter/counter.yaml.tmpl +++ b/demos/counter/counter.yaml.tmpl @@ -29,6 +29,23 @@ metadata: spec: replicas: 5 ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + template: + # Per-worker resources: these size the worker POD and advertise its + # scheduling capacity (the per-actor ceiling). The sandbox itself is sized by + # the ActorTemplate's spec.resources below, not by these (see internal/sizing). + # + # Capacity comes from the limits (workerCapacity reads limits only), while the + # kube-scheduler packs nodes by the requests — so the CPU request is set well + # below the limit to keep a 5-replica pool placeable on a small node (a single + # kind node in CI) without changing what an actor is allowed to use. Memory is + # not compressible, so its request matches the limit. + resources: + limits: + cpu: "1" + memory: 1Gi + requests: + cpu: 250m + memory: 1Gi --- @@ -52,6 +69,13 @@ ${VALIDATE_EXISTING_FILE_PATH_ARG} - name: data mountPath: /home/counter ${EXTERNAL_VOLUME_MOUNTS} + # Sandbox size: an actor occupies its whole worker, so size it at (or below) + # the pool's per-worker capacity above. ateom writes these to the OCI spec + # (cgroup CPU quota + memory limit) — see internal/sizing. + resources: + limits: + cpu: "1" + memory: 512Mi workerSelector: matchLabels: workload: counter diff --git a/demos/sandbox/sandbox.yaml.tmpl b/demos/sandbox/sandbox.yaml.tmpl index b9a868f04..344804b82 100644 --- a/demos/sandbox/sandbox.yaml.tmpl +++ b/demos/sandbox/sandbox.yaml.tmpl @@ -27,6 +27,23 @@ metadata: spec: replicas: 2 ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + template: + # Per-worker resources: these size the worker POD and advertise its + # scheduling capacity (the per-actor ceiling). The sandbox itself is sized by + # the ActorTemplate's spec.resources below, not by these (see internal/sizing). + # + # Capacity comes from the limits (workerCapacity reads limits only), while the + # kube-scheduler packs nodes by the requests — so the CPU request is set well + # below the limit to keep the pool placeable on a small node without changing + # what an actor is allowed to use. Memory is not compressible, so its request + # matches the limit. + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 500m + memory: 2Gi --- apiVersion: ate.dev/v1alpha1 kind: ActorTemplate @@ -43,5 +60,12 @@ spec: env: - name: PORT value: "80" + # Sandbox size: an actor occupies its whole worker, so size it at (or below) + # the pool's per-worker capacity above. ateom writes these to the OCI spec + # (cgroup CPU quota + memory limit) — see internal/sizing. + resources: + limits: + cpu: "2" + memory: 1Gi snapshotsConfig: location: gs://${BUCKET_NAME}/ate-demo-sandbox/ diff --git a/docs/api-guide.md b/docs/api-guide.md index e0d8f5f9f..dfff194c5 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -26,6 +26,13 @@ The `WorkerPool` defines the pool of physical "warm" compute capacity. It manage | `nodeAffinity` | `NodeAffinity` | `spec.affinity.nodeAffinity` | | `resources` | `ResourceRequirements` | `spec.containers[].resources` | +#### Worker Capacity (`spec.template.resources`) + +Setting `resources.limits` (CPU and Memory) on a `WorkerPool` establishes each worker pod's **capacity** — the envelope available to host an actor sandbox, taken from the `ateom` container's limits. The scheduler only places an actor on a worker whose capacity is `>=` the actor's declared resource limits (see [Sandbox Right-Sizing](#sandbox-right-sizing-specresources) on the `ActorTemplate`). + +- Size a pool's `limits` to the largest actor it should host. An actor occupies its whole worker, so worker capacity is the per-actor ceiling, not a shared budget. +- Capacity is advisory for placement only: a worker that declares no CPU/memory limit reports zero capacity for that dimension, which the scheduler treats as **unconstrained** (placement is never blocked by missing data). The actual sandbox size still comes from the `ActorTemplate`. + ### Example ```yaml @@ -145,11 +152,24 @@ The `ActorTemplate` defines the code, environment, and state-management policies | `workerSelector` | `*LabelSelector` | Optional. Gates which `WorkerPool`s actors from this template may use, by matching against each pool's labels. If unset, all pools are eligible (subject to the actor's own `worker_selector`). | | `snapshotsConfig` | `SnapshotsConfig` | **Required.** The base object-storage location snapshots are written under, plus the pause/commit/resume scopes. See [Snapshot Storage Layout](#snapshot-storage-layout). | | `volumes` | `[]Volume` | Optional. Volumes the containers may mount, each either a `durableDir` or an `externalVolumeTemplate`. Every declared volume must be mounted by at least one container. A `microvm` template may declare several `durableDir` volumes; a `gvisor` template is limited to one, and `externalVolumeTemplate` is `gvisor`-only. | +| `resources` | `*ResourceRequirements` | Optional. Declares each actor's compute size via `limits` — see [Sandbox Right-Sizing](#sandbox-right-sizing-specresources). Immutable, like the rest of the spec. | The sandbox itself — the binaries (e.g. the gVisor `runsc` binary) and the `pauseImage` holding the sandbox's namespaces — is **not configured on the `ActorTemplate`**. It is resolved from the referenced `WorkerPool`'s [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) — by name (`workerPool.spec.sandboxConfigName`) or, by default, the cluster default `SandboxConfig` for the pool's `sandboxClass`. Because a snapshot is not restorable across sandbox runtimes, `sandboxClass` is a **hard scheduling gate**: an actor is only ever placed on a `WorkerPool` of the matching class. It is AND'd with `workerSelector` (and the actor's `worker_selector`), which can only narrow the eligible pools further. It defaults to `gvisor` and, like the rest of the spec, is immutable, so each template's class is fixed at creation. +### Sandbox Right-Sizing (`spec.resources`) + +Unlike a Pod, an actor is sized by its **`limits`** (CPU and Memory): the size is a property of the template, baked into snapshots, so it lives on the immutable `ActorTemplate` spec. Declared limits do three things: + +1. **Size the sandbox.** The limits are supplied to the sandbox over the actor RPCs (control plane → atelet → ateom) and applied to the container OCI spec: + - **gVisor (`ateom-gvisor`)** — `limits.cpu` sets the cgroup v2 CPU quota (`cpu.max`) and the Sentry vCPU count (`--cpu-num-from-quota`); `limits.memory` sets the cgroup v2 memory limit (`memory.max`) and bounds the virtual total memory the sandbox reports (so JVM/Go do not over-allocate from host RAM). + - **Micro-VM (`ateom-microvm`)** — `limits.cpu` sets Cloud Hypervisor `BootVcpus` / `MaxVcpus` (rounded up to whole vCPUs); `limits.memory` sets guest RAM, reserving a small configurable margin (default 256 MiB, `--vmm-mem-reserve-mib`) for the VMM and virtiofsd so the pod cgroup does not OOM. +2. **Gate scheduling.** An actor is only placed on a `WorkerPool` whose [worker capacity](#worker-capacity-spectemplateresources) is `>=` these limits. +3. **Fall back to runtime defaults.** A zero or absent limit leaves that dimension at the runtime default — unlimited for gVisor, the kata config for the micro-VM. + +`requests` are not consulted today (an actor occupies its whole worker). Because the size is baked into snapshots, a **micro-VM FULL-scope restore reuses the size in the snapshot**; changing an actor's limits takes effect on its next cold boot. + Container environment variables support literal `value` entries only. Values are not interpolated (`$(VAR)` references are not expanded), and Kubernetes `envFrom`/`valueFrom` sources are not supported. ### Workload Connectivity (Uniform DNS) diff --git a/internal/e2e/fixtures/probe/main.go b/internal/e2e/fixtures/probe/main.go index 6927a1d04..ec2192bea 100644 --- a/internal/e2e/fixtures/probe/main.go +++ b/internal/e2e/fixtures/probe/main.go @@ -21,10 +21,14 @@ package main import ( + "bufio" "encoding/json" "log" "net/http" "os" + "runtime" + "strconv" + "strings" ) // identityFile is the actor-id file inside the identity directory atelet @@ -48,6 +52,66 @@ func whoami(w http.ResponseWriter, _ *http.Request) { writeJSON(w, resp) } +// resources reports the compute envelope the actor observes from inside the +// sandbox, so the sizing e2e suite can assert the actor's declared limits +// actually shaped the runtime. +// +// - num_cpu is runtime.NumCPU(): for the gVisor runtime this is the sentry's +// vCPU count, provisioned from the CPU limit via runsc --cpu-num-from-quota, +// so it equals ceil(limits.cpu). +// - mem_total_bytes is MemTotal from /proc/meminfo: the memory the sandbox +// believes it has, bounded by limits.memory. +// - cpu_max / memory_max are the raw cgroup v2 files, reported best-effort for +// debugging; presence and format vary by runtime. +func resources(w http.ResponseWriter, _ *http.Request) { + resp := map[string]any{"num_cpu": runtime.NumCPU()} + + if v, err := memTotalBytes(); err == nil { + resp["mem_total_bytes"] = v + } else { + resp["mem_total_error"] = err.Error() + } + if b, err := os.ReadFile("/sys/fs/cgroup/cpu.max"); err == nil { + resp["cpu_max"] = strings.TrimSpace(string(b)) + } + if b, err := os.ReadFile("/sys/fs/cgroup/memory.max"); err == nil { + resp["memory_max"] = strings.TrimSpace(string(b)) + } + + writeJSON(w, resp) +} + +// memTotalBytes parses MemTotal (reported in kB) from /proc/meminfo. +func memTotalBytes() (int64, error) { + f, err := os.Open("/proc/meminfo") + if err != nil { + return 0, err + } + defer f.Close() + + sc := bufio.NewScanner(f) + for sc.Scan() { + line := sc.Text() + rest, ok := strings.CutPrefix(line, "MemTotal:") + if !ok { + continue + } + fields := strings.Fields(rest) // e.g. "524288 kB" + if len(fields) == 0 { + break + } + kb, err := strconv.ParseInt(fields[0], 10, 64) + if err != nil { + return 0, err + } + return kb * 1024, nil + } + if err := sc.Err(); err != nil { + return 0, err + } + return 0, os.ErrNotExist +} + func writeJSON(w http.ResponseWriter, v any) { w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(v); err != nil { @@ -58,6 +122,7 @@ func writeJSON(w http.ResponseWriter, v any) { func main() { mux := http.NewServeMux() mux.HandleFunc("/whoami", whoami) + mux.HandleFunc("/resources", resources) mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) const addr = ":80" diff --git a/internal/e2e/fixtures/probe/probe-sized.yaml.tmpl b/internal/e2e/fixtures/probe/probe-sized.yaml.tmpl new file mode 100644 index 000000000..fdf0f9a7d --- /dev/null +++ b/internal/e2e/fixtures/probe/probe-sized.yaml.tmpl @@ -0,0 +1,69 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Sized variant of the probe fixture: the ActorTemplate declares +# spec.resources.limits, so the sizing e2e suite can assert the actor's +# sandbox is shaped to those limits. Reuses the probe image (it serves the +# /resources endpoint). Kept in its own namespace so it never collides with +# the plain probe fixture. + +apiVersion: v1 +kind: Namespace +metadata: + name: ate-e2e-sizing + +--- + +apiVersion: ate.dev/v1alpha1 +kind: WorkerPool +metadata: + name: probe-sized + namespace: ate-e2e-sizing + labels: + workload: probe-sized +spec: + replicas: 3 + ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + +--- + +apiVersion: ate.dev/v1alpha1 +kind: ActorTemplate +metadata: + name: probe-sized + namespace: ate-e2e-sizing +spec: + containers: + - name: probe + image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/probe + command: ["/ko-app/probe"] + # Gate the golden snapshot on the probe actually serving, so the single + # (un-retried) GET /resources the suite makes after resuming cannot race a + # sandbox that was checkpointed before the listener was up. + readyz: + httpGet: + path: /healthz + port: 80 + # The feature under test: these limits size the sandbox (and, when the worker + # advertises capacity, gate scheduling). CPU=2 makes NumCPU() inside the + # gVisor sandbox a distinct, assertable value. + resources: + limits: + cpu: "2" + memory: 512Mi + workerSelector: + matchLabels: + workload: probe-sized + snapshotsConfig: + location: gs://${BUCKET_NAME}/ate-e2e-sizing/ diff --git a/internal/e2e/suites/sizing/sizing_test.go b/internal/e2e/suites/sizing/sizing_test.go new file mode 100644 index 000000000..fe85cacdd --- /dev/null +++ b/internal/e2e/suites/sizing/sizing_test.go @@ -0,0 +1,203 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sizing + +import ( + "context" + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/agent-substrate/substrate/internal/e2e" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + sizingNamespace = "ate-e2e-sizing" + sizingTemplate = "probe-sized" + + // The limits declared in probe-sized.yaml.tmpl. Keep these in sync with the + // manifest: the whole point of the suite is to assert the sandbox observes + // exactly what the ActorTemplate declared. + wantCPU = 2 + wantMemBytes = 512 * 1024 * 1024 // 512Mi +) + +// resourcesResponse mirrors the /resources endpoint of the probe fixture. +type resourcesResponse struct { + NumCPU int `json:"num_cpu"` + MemTotalBytes int64 `json:"mem_total_bytes"` + MemTotalError string `json:"mem_total_error"` + CPUMax string `json:"cpu_max"` + MemoryMax string `json:"memory_max"` +} + +// TestActorSizing_SandboxObservesDeclaredLimits is the end-to-end gate for the +// resource-limits redesign: an ActorTemplate that declares spec.resources.limits +// must produce a sandbox sized to those limits. The plumbing unit tests prove +// the limits reach the OCI spec; this proves the running sandbox actually +// honors them, by resuming an actor and asking it (via the probe /resources +// endpoint) what compute envelope it sees from the inside. +// +// gVisor is the default (and only) runtime in the macOS/colima kind +// environment, so the assertions target what runsc --cpu-num-from-quota and the +// cgroup memory limit produce inside the sentry. +func TestActorSizing_SandboxObservesDeclaredLimits(t *testing.T) { + env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") + if err != nil { + t.Fatalf("CheckEnv failed: %v", err) + } + ctx := context.Background() + clients := e2e.GetClients() + + deploySizedProbe(t, env["BUCKET_NAME"]) + waitForTemplateReady(t, ctx, clients) + + const id = "sized-actor" + createAndResumeActor(t, ctx, clients, id) + + rc, err := e2e.NewRouterClient(ctx) + if err != nil { + t.Fatalf("NewRouterClient: %v", err) + } + defer rc.Close() + + got := getResources(t, ctx, rc, id) + t.Logf("sandbox /resources: num_cpu=%d mem_total_bytes=%d cpu_max=%q memory_max=%q mem_total_error=%q", + got.NumCPU, got.MemTotalBytes, got.CPUMax, got.MemoryMax, got.MemTotalError) + + // CPU: runsc provisions the sentry's vCPU count from the CPU quota + // (--cpu-num-from-quota), so the sandbox must see exactly the declared limit. + if got.NumCPU != wantCPU { + t.Errorf("sandbox NumCPU = %d, want %d (declared limits.cpu=%d) — sandbox not sized to actor limits", got.NumCPU, wantCPU, wantCPU) + } + + // Memory: the sandbox must be bounded by the declared limit. gVisor may + // report slightly under the limit (reserved overhead) but must never see + // more; a value near the node's full RAM means the limit was not applied. + // Allow 10% headroom above the limit for accounting differences. + if got.MemTotalError != "" { + t.Errorf("probe could not read MemTotal: %s", got.MemTotalError) + } else if got.MemTotalBytes > wantMemBytes*11/10 { + t.Errorf("sandbox MemTotal = %d bytes, want <= ~%d (declared limits.memory=512Mi) — memory limit not applied", got.MemTotalBytes, wantMemBytes) + } else if got.MemTotalBytes < wantMemBytes/2 { + t.Errorf("sandbox MemTotal = %d bytes, unexpectedly far below the declared 512Mi limit", got.MemTotalBytes) + } +} + +func deploySizedProbe(t *testing.T, bucket string) { + t.Helper() + root, err := e2e.FindRepoRoot() + if err != nil { + t.Fatalf("FindRepoRoot: %v", err) + } + + // Render the manifest template to a file so both apply and delete can + // consume it without any shell involved (mirrors the identity suite). + tmpl, err := os.ReadFile(filepath.Join(root, "internal/e2e/fixtures/probe/probe-sized.yaml.tmpl")) + if err != nil { + t.Fatalf("reading sized probe manifest template: %v", err) + } + manifest := filepath.Join(t.TempDir(), "probe-sized.yaml") + rendered := strings.ReplaceAll(string(tmpl), "${BUCKET_NAME}", bucket) + if err := os.WriteFile(manifest, []byte(rendered), 0o644); err != nil { + t.Fatalf("writing rendered sized probe manifest: %v", err) + } + + // Build/push the probe image and apply through the repo's pinned ko. See the + // identity suite's deployProbe for why KO_CONFIG_PATH and the trailing + // `-- --context=...` are required. + applyArgs := []string{"ko", "apply", "-f", manifest} + if e2e.KubeContext != "" { + applyArgs = append(applyArgs, "--", "--context="+e2e.KubeContext) + } + e2e.RunCmdWithEnv(t, []string{"KO_CONFIG_PATH=" + root}, filepath.Join(root, "hack/run-tool.sh"), applyArgs...) + + t.Cleanup(func() { + delArgs := []string{"delete", "--ignore-not-found", "-f", manifest} + if e2e.KubeContext != "" { + delArgs = append([]string{"--context=" + e2e.KubeContext}, delArgs...) + } + e2e.RunCmd(t, "kubectl", delArgs...) + }) +} + +func waitForTemplateReady(t *testing.T, ctx context.Context, clients *e2e.Clients) { + t.Helper() + deadline := time.Now().Add(5 * time.Minute) + for time.Now().Before(deadline) { + at, err := clients.SubstrateK8s.ApiV1alpha1().ActorTemplates(sizingNamespace).Get(ctx, sizingTemplate, metav1.GetOptions{}) + if err == nil { + switch at.Status.Phase { + case v1alpha1.PhaseReady: + t.Logf("sized probe ActorTemplate ready, golden=%s", at.Status.GoldenActorID) + return + case v1alpha1.PhaseFailed: + t.Fatalf("sized probe ActorTemplate entered PhaseFailed") + } + } + time.Sleep(2 * time.Second) + } + t.Fatalf("timed out waiting for sized probe ActorTemplate to be Ready") +} + +func createAndResumeActor(t *testing.T, ctx context.Context, clients *e2e.Clients, id string) { + t.Helper() + // CreateActor requires the atespace to exist first. + _, _ = clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: sizingNamespace}}}) + if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: sizingNamespace, Name: id}, + ActorTemplateNamespace: sizingNamespace, + ActorTemplateName: sizingTemplate, + }}); err != nil { + t.Fatalf("CreateActor %q: %v", id, err) + } + t.Cleanup(func() { + // DeleteActor requires the actor to be suspended. + _, _ = clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: &ateapipb.ObjectRef{Atespace: sizingNamespace, Name: id}}) + _, _ = clients.SubstrateAPI.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: &ateapipb.ObjectRef{Atespace: sizingNamespace, Name: id}}) + }) + + // Resume from the golden snapshot (the restore path, not --boot). + if _, err := clients.SubstrateAPI.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: &ateapipb.ObjectRef{Atespace: sizingNamespace, Name: id}}); err != nil { + t.Fatalf("ResumeActor %q: %v", id, err) + } +} + +func getResources(t *testing.T, ctx context.Context, rc *e2e.RouterClient, id string) resourcesResponse { + t.Helper() + resp, err := rc.Get(ctx, resources.ActorRef{Atespace: sizingNamespace, Name: id}, "/resources") + if err != nil { + t.Fatalf("GET /resources for %q: %v", id, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("GET /resources for %q: status %d, body %q", id, resp.StatusCode, body) + } + var out resourcesResponse + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatalf("decoding /resources for %q: %v", id, err) + } + return out +} diff --git a/internal/e2e/suites/sizing/testmain_test.go b/internal/e2e/suites/sizing/testmain_test.go new file mode 100644 index 000000000..5ba57d803 --- /dev/null +++ b/internal/e2e/suites/sizing/testmain_test.go @@ -0,0 +1,26 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sizing + +import ( + "os" + "testing" + + "github.com/agent-substrate/substrate/internal/e2e" +) + +func TestMain(m *testing.M) { + os.Exit(e2e.RunTestMain(m)) +} diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index c6a5b5814..dab3709fb 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -316,6 +316,11 @@ type RunRequest struct { SandboxAssets *SandboxAssets `protobuf:"bytes,8,opt,name=sandbox_assets,json=sandboxAssets,proto3" json:"sandbox_assets,omitempty"` // When absent, actor traffic uses direct egress instead of atunnel. EgressGateway *EgressGateway `protobuf:"bytes,9,opt,name=egress_gateway,json=egressGateway,proto3,oneof" json:"egress_gateway,omitempty"` + // The actor's declared size, from the ActorTemplate's resource limits. atelet + // passes these through to the sandbox so it is sized to the actor (not the + // whole host or worker pod). Zero means "unset": keep the runtime default. + CpuMilli int64 `protobuf:"varint,10,opt,name=cpu_milli,json=cpuMilli,proto3" json:"cpu_milli,omitempty"` // CPU limit in millicores (1000 = one core). + MemoryBytes int64 `protobuf:"varint,11,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory limit in bytes. unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -413,6 +418,20 @@ func (x *RunRequest) GetEgressGateway() *EgressGateway { return nil } +func (x *RunRequest) GetCpuMilli() int64 { + if x != nil { + return x.CpuMilli + } + return 0 +} + +func (x *RunRequest) GetMemoryBytes() int64 { + if x != nil { + return x.MemoryBytes + } + return 0 +} + // EgressGateway configures tunneled egress for one actor activation. type EgressGateway struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1687,6 +1706,12 @@ type RestoreRequest struct { GoldenSnapshotUri string `protobuf:"bytes,12,opt,name=golden_snapshot_uri,json=goldenSnapshotUri,proto3" json:"golden_snapshot_uri,omitempty"` // When absent, actor traffic uses direct egress instead of atunnel. EgressGateway *EgressGateway `protobuf:"bytes,13,opt,name=egress_gateway,json=egressGateway,proto3,oneof" json:"egress_gateway,omitempty"` + // The actor's declared size, from the ActorTemplate's resource limits. For + // gVisor and micro-VM DATA-scope restores the sandbox is (re)sized to these; + // for a FULL micro-VM restore the size baked into the snapshot wins. Zero + // means "unset": keep the runtime default. + CpuMilli int64 `protobuf:"varint,14,opt,name=cpu_milli,json=cpuMilli,proto3" json:"cpu_milli,omitempty"` // CPU limit in millicores (1000 = one core). + MemoryBytes int64 `protobuf:"varint,15,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory limit in bytes. unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1823,6 +1848,20 @@ func (x *RestoreRequest) GetEgressGateway() *EgressGateway { return nil } +func (x *RestoreRequest) GetCpuMilli() int64 { + if x != nil { + return x.CpuMilli + } + return 0 +} + +func (x *RestoreRequest) GetMemoryBytes() int64 { + if x != nil { + return x.MemoryBytes + } + return 0 +} + type isRestoreRequest_Config interface { isRestoreRequest_Config() } @@ -1884,7 +1923,7 @@ const file_atelet_proto_rawDesc = "" + "\x1bcertificate_signing_request\x18\x01 \x01(\fR\x19certificateSigningRequest\x12,\n" + "\x12expected_actor_uid\x18\x02 \x01(\tR\x10expectedActorUid\"M\n" + "\x1cMintActorCertificateResponse\x12-\n" + - "\x12actor_certificates\x18\x01 \x03(\fR\x11actorCertificates\"\xb6\x03\n" + + "\x12actor_certificates\x18\x01 \x03(\fR\x11actorCertificates\"\xf6\x03\n" + "\n" + "RunRequest\x12(\n" + "\x10target_ateom_uid\x18\x01 \x01(\tR\x0etargetAteomUid\x12\x1a\n" + @@ -1896,7 +1935,10 @@ const file_atelet_proto_rawDesc = "" + "\x13actor_template_name\x18\x06 \x01(\tR\x11actorTemplateName\x12(\n" + "\x04spec\x18\a \x01(\v2\x14.atelet.WorkloadSpecR\x04spec\x12<\n" + "\x0esandbox_assets\x18\b \x01(\v2\x15.atelet.SandboxAssetsR\rsandboxAssets\x12A\n" + - "\x0eegress_gateway\x18\t \x01(\v2\x15.atelet.EgressGatewayH\x00R\regressGateway\x88\x01\x01B\x11\n" + + "\x0eegress_gateway\x18\t \x01(\v2\x15.atelet.EgressGatewayH\x00R\regressGateway\x88\x01\x01\x12\x1b\n" + + "\tcpu_milli\x18\n" + + " \x01(\x03R\bcpuMilli\x12!\n" + + "\fmemory_bytes\x18\v \x01(\x03R\vmemoryBytesB\x11\n" + "\x0f_egress_gateway\")\n" + "\rEgressGateway\x12\x18\n" + "\aaddress\x18\x01 \x01(\tR\aaddress\"5\n" + @@ -1991,7 +2033,7 @@ const file_atelet_proto_rawDesc = "" + "\x13local_snapshot_name\x18\x06 \x01(\tR\x11localSnapshotName\x128\n" + "\x18destination_snapshot_uri\x18\a \x01(\tR\x16destinationSnapshotUri\x12:\n" + "\rdesired_scope\x18\b \x01(\x0e2\x15.atelet.SnapshotScopeR\fdesiredScope\" \n" + - "\x1eUploadPausedCheckpointResponse\"\xae\x05\n" + + "\x1eUploadPausedCheckpointResponse\"\xee\x05\n" + "\x0eRestoreRequest\x12(\n" + "\x10target_ateom_uid\x18\x01 \x01(\tR\x0etargetAteomUid\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x12\x1d\n" + @@ -2007,7 +2049,9 @@ const file_atelet_proto_rawDesc = "" + " \x01(\v2'.atelet.ExternalCheckpointConfigurationH\x00R\x0eexternalConfig\x12+\n" + "\x05scope\x18\v \x01(\x0e2\x15.atelet.SnapshotScopeR\x05scope\x12.\n" + "\x13golden_snapshot_uri\x18\f \x01(\tR\x11goldenSnapshotUri\x12A\n" + - "\x0eegress_gateway\x18\r \x01(\v2\x15.atelet.EgressGatewayH\x01R\regressGateway\x88\x01\x01B\b\n" + + "\x0eegress_gateway\x18\r \x01(\v2\x15.atelet.EgressGatewayH\x01R\regressGateway\x88\x01\x01\x12\x1b\n" + + "\tcpu_milli\x18\x0e \x01(\x03R\bcpuMilli\x12!\n" + + "\fmemory_bytes\x18\x0f \x01(\x03R\vmemoryBytesB\b\n" + "\x06configB\x11\n" + "\x0f_egress_gateway\"\x11\n" + "\x0fRestoreResponse*`\n" + diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index 92263d230..6b5a68c2e 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -77,6 +77,12 @@ message RunRequest { // When absent, actor traffic uses direct egress instead of atunnel. optional EgressGateway egress_gateway = 9; + + // The actor's declared size, from the ActorTemplate's resource limits. atelet + // passes these through to the sandbox so it is sized to the actor (not the + // whole host or worker pod). Zero means "unset": keep the runtime default. + int64 cpu_milli = 10; // CPU limit in millicores (1000 = one core). + int64 memory_bytes = 11; // Memory limit in bytes. } // EgressGateway configures tunneled egress for one actor activation. @@ -327,6 +333,13 @@ message RestoreRequest { // When absent, actor traffic uses direct egress instead of atunnel. optional EgressGateway egress_gateway = 13; + + // The actor's declared size, from the ActorTemplate's resource limits. For + // gVisor and micro-VM DATA-scope restores the sandbox is (re)sized to these; + // for a FULL micro-VM restore the size baked into the snapshot wins. Zero + // means "unset": keep the runtime default. + int64 cpu_milli = 14; // CPU limit in millicores (1000 = one core). + int64 memory_bytes = 15; // Memory limit in bytes. } message RestoreResponse { diff --git a/internal/proto/ateompb/ateom.pb.go b/internal/proto/ateompb/ateom.pb.go index a0c83adb8..c865d0a1c 100644 --- a/internal/proto/ateompb/ateom.pb.go +++ b/internal/proto/ateompb/ateom.pb.go @@ -284,6 +284,12 @@ type RunWorkloadRequest struct { RuntimeAssetPaths map[string]string `protobuf:"bytes,8,rep,name=runtime_asset_paths,json=runtimeAssetPaths,proto3" json:"runtime_asset_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // When absent, actor traffic uses direct egress instead of atunnel. EgressGateway *EgressGateway `protobuf:"bytes,10,opt,name=egress_gateway,json=egressGateway,proto3,oneof" json:"egress_gateway,omitempty"` + // The actor's declared size, from the ActorTemplate's resource limits. ateom + // sizes the sandbox to these (cgroup caps via the OCI spec, and for the + // micro-VM the VM's vCPU count and memory). Zero means "unset": keep the + // runtime default (unlimited for gVisor, the kata config for the micro-VM). + CpuMilli int64 `protobuf:"varint,11,opt,name=cpu_milli,json=cpuMilli,proto3" json:"cpu_milli,omitempty"` // CPU limit in millicores (1000 = one core). + MemoryBytes int64 `protobuf:"varint,12,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory limit in bytes. unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -381,6 +387,20 @@ func (x *RunWorkloadRequest) GetEgressGateway() *EgressGateway { return nil } +func (x *RunWorkloadRequest) GetCpuMilli() int64 { + if x != nil { + return x.CpuMilli + } + return 0 +} + +func (x *RunWorkloadRequest) GetMemoryBytes() int64 { + if x != nil { + return x.MemoryBytes + } + return 0 +} + // EgressGateway configures tunneled egress for one actor activation. type EgressGateway struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -937,8 +957,14 @@ type RestoreWorkloadRequest struct { // Set only when scope is SNAPSHOT_SCOPE_DATA_ON_GOLDEN. Mirrors the // snapshot_uri contract (field 8). GoldenSnapshotUri string `protobuf:"bytes,13,opt,name=golden_snapshot_uri,json=goldenSnapshotUri,proto3" json:"golden_snapshot_uri,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // The actor's declared size, from the ActorTemplate's resource limits. Used to + // (re)size the sandbox on a DATA-scope restore (fresh guest container). On a + // FULL micro-VM restore the size baked into the snapshot is authoritative and + // these are ignored. Zero means "unset": keep the runtime default. + CpuMilli int64 `protobuf:"varint,14,opt,name=cpu_milli,json=cpuMilli,proto3" json:"cpu_milli,omitempty"` // CPU limit in millicores (1000 = one core). + MemoryBytes int64 `protobuf:"varint,15,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory limit in bytes. + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RestoreWorkloadRequest) Reset() { @@ -1055,6 +1081,20 @@ func (x *RestoreWorkloadRequest) GetGoldenSnapshotUri() string { return "" } +func (x *RestoreWorkloadRequest) GetCpuMilli() int64 { + if x != nil { + return x.CpuMilli + } + return 0 +} + +func (x *RestoreWorkloadRequest) GetMemoryBytes() int64 { + if x != nil { + return x.MemoryBytes + } + return 0 +} + type RestoreWorkloadResponse struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -1477,7 +1517,7 @@ var File_ateom_proto protoreflect.FileDescriptor const file_ateom_proto_rawDesc = "" + "\n" + - "\vateom.proto\x12\x05ateom\"\x9b\x04\n" + + "\vateom.proto\x12\x05ateom\"\xdb\x04\n" + "\x12RunWorkloadRequest\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1d\n" + "\n" + @@ -1490,7 +1530,9 @@ const file_ateom_proto_rawDesc = "" + "\x04spec\x18\a \x01(\v2\x13.ateom.WorkloadSpecR\x04spec\x12`\n" + "\x13runtime_asset_paths\x18\b \x03(\v20.ateom.RunWorkloadRequest.RuntimeAssetPathsEntryR\x11runtimeAssetPaths\x12@\n" + "\x0eegress_gateway\x18\n" + - " \x01(\v2\x14.ateom.EgressGatewayH\x00R\regressGateway\x88\x01\x01\x1aD\n" + + " \x01(\v2\x14.ateom.EgressGatewayH\x00R\regressGateway\x88\x01\x01\x12\x1b\n" + + "\tcpu_milli\x18\v \x01(\x03R\bcpuMilli\x12!\n" + + "\fmemory_bytes\x18\f \x01(\x03R\vmemoryBytes\x1aD\n" + "\x16RuntimeAssetPathsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01B\x11\n" + @@ -1535,7 +1577,7 @@ const file_ateom_proto_rawDesc = "" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"C\n" + "\x1aCheckpointWorkloadResponse\x12%\n" + - "\x0esnapshot_files\x18\x01 \x03(\tR\rsnapshotFiles\"\xa2\x05\n" + + "\x0esnapshot_files\x18\x01 \x03(\tR\rsnapshotFiles\"\xe2\x05\n" + "\x16RestoreWorkloadRequest\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1d\n" + "\n" + @@ -1551,7 +1593,9 @@ const file_ateom_proto_rawDesc = "" + "\x05scope\x18\n" + " \x01(\x0e2\x14.ateom.SnapshotScopeR\x05scope\x12@\n" + "\x0eegress_gateway\x18\f \x01(\v2\x14.ateom.EgressGatewayH\x00R\regressGateway\x88\x01\x01\x12.\n" + - "\x13golden_snapshot_uri\x18\r \x01(\tR\x11goldenSnapshotUri\x1aD\n" + + "\x13golden_snapshot_uri\x18\r \x01(\tR\x11goldenSnapshotUri\x12\x1b\n" + + "\tcpu_milli\x18\x0e \x01(\x03R\bcpuMilli\x12!\n" + + "\fmemory_bytes\x18\x0f \x01(\x03R\vmemoryBytes\x1aD\n" + "\x16RuntimeAssetPathsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01B\x11\n" + diff --git a/internal/proto/ateompb/ateom.proto b/internal/proto/ateompb/ateom.proto index 877f8fe47..9ec80a232 100644 --- a/internal/proto/ateompb/ateom.proto +++ b/internal/proto/ateompb/ateom.proto @@ -119,6 +119,13 @@ message RunWorkloadRequest { // When absent, actor traffic uses direct egress instead of atunnel. optional EgressGateway egress_gateway = 10; + + // The actor's declared size, from the ActorTemplate's resource limits. ateom + // sizes the sandbox to these (cgroup caps via the OCI spec, and for the + // micro-VM the VM's vCPU count and memory). Zero means "unset": keep the + // runtime default (unlimited for gVisor, the kata config for the micro-VM). + int64 cpu_milli = 11; // CPU limit in millicores (1000 = one core). + int64 memory_bytes = 12; // Memory limit in bytes. } // EgressGateway configures tunneled egress for one actor activation. @@ -262,6 +269,13 @@ message RestoreWorkloadRequest { // Set only when scope is SNAPSHOT_SCOPE_DATA_ON_GOLDEN. Mirrors the // snapshot_uri contract (field 8). string golden_snapshot_uri = 13; + + // The actor's declared size, from the ActorTemplate's resource limits. Used to + // (re)size the sandbox on a DATA-scope restore (fresh guest container). On a + // FULL micro-VM restore the size baked into the snapshot is authoritative and + // these are ignored. Zero means "unset": keep the runtime default. + int64 cpu_milli = 14; // CPU limit in millicores (1000 = one core). + int64 memory_bytes = 15; // Memory limit in bytes. } message RestoreWorkloadResponse { diff --git a/internal/sizing/sizing.go b/internal/sizing/sizing.go new file mode 100644 index 000000000..ff1f46865 --- /dev/null +++ b/internal/sizing/sizing.go @@ -0,0 +1,105 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package sizing right-sizes a sandbox to the actor's declared resource limits. +// The bulk of right-sizing is writing the correct cgroup values, which is +// identical for the gVisor and micro-VM runtimes, so both ateom binaries share +// this package: the actor's limits arrive over the ateom RPCs (RunWorkload / +// RestoreWorkload) and ApplyToOCISpec writes them into the container OCI spec. +// runsc then applies them to the host cgroup leaf (gVisor) and the kata-agent +// applies them to the guest cgroup (micro-VM). The micro-VM additionally sizes +// the VM itself from the same SandboxSize (see VCPUs). +package sizing + +import ( + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +const ( + // cpuQuotaPeriodMicros is the cgroup v2 cpu.max period (100ms, the kernel + // default) against which the CPU quota is expressed. + cpuQuotaPeriodMicros = 100000 +) + +// SandboxSize is the sandbox's target size, derived from the actor's declared +// resource limits. A zero field means "unset": the caller keeps its own default +// (the kata config for the micro-VM, unlimited for gVisor). +type SandboxSize struct { + // MilliCPU is the CPU limit in millicores (1000 = one core), or 0 if unset. + MilliCPU int64 + // MemoryBytes is the memory limit in bytes, or 0 if unset. + MemoryBytes int64 +} + +// FromLimits builds a SandboxSize from an actor's declared limits (millicores and +// bytes) as carried on the ateom RPCs. It is runtime-agnostic; both ateom-gvisor +// and ateom-microvm call it. Negative values are clamped to zero ("unset"). +func FromLimits(milliCPU, memoryBytes int64) SandboxSize { + if milliCPU < 0 { + milliCPU = 0 + } + if memoryBytes < 0 { + memoryBytes = 0 + } + return SandboxSize{MilliCPU: milliCPU, MemoryBytes: memoryBytes} +} + +// VCPUs converts the CPU limit to a whole vCPU count for the micro-VM, rounding +// up so a fractional limit still yields a usable core (minimum 1 when a limit is +// set). Returns 0 when the CPU limit is unset, letting the caller keep its +// default. +func (s SandboxSize) VCPUs() int { + if s.MilliCPU <= 0 { + return 0 + } + v := (s.MilliCPU + 999) / 1000 + if v < 1 { + v = 1 + } + return int(v) +} + +// ApplyToOCISpec writes the pod's CPU/memory limits into the container OCI spec's +// linux.resources so the sandbox cgroup is created with the right values. This is +// the piece shared by both runtimes: runsc applies it to the host cgroup leaf +// (gVisor) and the kata-agent applies it to the guest cgroup (micro-VM). Fields +// that are unset in SandboxSize are left untouched, preserving any existing values +// (e.g. the micro-VM's device allowlist and CPU shares). +func (s SandboxSize) ApplyToOCISpec(spec *specs.Spec) { + if s.MilliCPU <= 0 && s.MemoryBytes <= 0 { + return + } + if spec.Linux == nil { + spec.Linux = &specs.Linux{} + } + if spec.Linux.Resources == nil { + spec.Linux.Resources = &specs.LinuxResources{} + } + if s.MilliCPU > 0 { + if spec.Linux.Resources.CPU == nil { + spec.Linux.Resources.CPU = &specs.LinuxCPU{} + } + period := uint64(cpuQuotaPeriodMicros) + quota := s.MilliCPU * cpuQuotaPeriodMicros / 1000 + spec.Linux.Resources.CPU.Period = &period + spec.Linux.Resources.CPU.Quota = "a + } + if s.MemoryBytes > 0 { + if spec.Linux.Resources.Memory == nil { + spec.Linux.Resources.Memory = &specs.LinuxMemory{} + } + limit := s.MemoryBytes + spec.Linux.Resources.Memory.Limit = &limit + } +} diff --git a/internal/sizing/sizing_test.go b/internal/sizing/sizing_test.go new file mode 100644 index 000000000..1c9ac24fc --- /dev/null +++ b/internal/sizing/sizing_test.go @@ -0,0 +1,103 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sizing + +import ( + "testing" + + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +func TestFromLimits(t *testing.T) { + got := FromLimits(1500, 2147483648) + if got.MilliCPU != 1500 || got.MemoryBytes != 2147483648 { + t.Fatalf("FromLimits() = %+v", got) + } +} + +func TestFromLimitsClampsNegative(t *testing.T) { + got := FromLimits(-5, -1) + if got.MilliCPU != 0 || got.MemoryBytes != 0 { + t.Fatalf("FromLimits() = %+v, want zero", got) + } +} + +func TestVCPUs(t *testing.T) { + cases := []struct { + milli int64 + want int + }{ + {0, 0}, + {1, 1}, + {999, 1}, + {1000, 1}, + {1001, 2}, + {2500, 3}, + {4000, 4}, + } + for _, c := range cases { + if got := (SandboxSize{MilliCPU: c.milli}).VCPUs(); got != c.want { + t.Errorf("VCPUs(%d) = %d, want %d", c.milli, got, c.want) + } + } +} + +func TestApplyToOCISpec(t *testing.T) { + spec := &specs.Spec{} + (SandboxSize{MilliCPU: 2000, MemoryBytes: 1073741824}).ApplyToOCISpec(spec) + + if spec.Linux == nil || spec.Linux.Resources == nil { + t.Fatal("resources not set") + } + cpu := spec.Linux.Resources.CPU + if cpu == nil || cpu.Quota == nil || cpu.Period == nil { + t.Fatal("cpu not set") + } + if *cpu.Period != cpuQuotaPeriodMicros || *cpu.Quota != 2*cpuQuotaPeriodMicros { + t.Errorf("cpu = quota %d period %d", *cpu.Quota, *cpu.Period) + } + mem := spec.Linux.Resources.Memory + if mem == nil || mem.Limit == nil || *mem.Limit != 1073741824 { + t.Errorf("memory limit not set correctly: %+v", mem) + } +} + +func TestApplyToOCISpecPreservesExistingAndSkipsUnset(t *testing.T) { + shares := uint64(1024) + spec := &specs.Spec{Linux: &specs.Linux{Resources: &specs.LinuxResources{ + CPU: &specs.LinuxCPU{Shares: &shares}, + }}} + // Only memory set; CPU limit unset must not clobber existing shares and must + // not add a quota. + (SandboxSize{MemoryBytes: 512}).ApplyToOCISpec(spec) + + if spec.Linux.Resources.CPU.Shares == nil || *spec.Linux.Resources.CPU.Shares != 1024 { + t.Error("existing cpu shares clobbered") + } + if spec.Linux.Resources.CPU.Quota != nil { + t.Error("cpu quota set despite unset MilliCPU") + } + if spec.Linux.Resources.Memory == nil || *spec.Linux.Resources.Memory.Limit != 512 { + t.Error("memory limit not applied") + } +} + +func TestApplyToOCISpecNoopWhenEmpty(t *testing.T) { + spec := &specs.Spec{} + (SandboxSize{}).ApplyToOCISpec(spec) + if spec.Linux != nil { + t.Error("empty SandboxSize mutated spec") + } +} diff --git a/manifests/ate-install/generated/ate.dev_actortemplates.yaml b/manifests/ate-install/generated/ate.dev_actortemplates.yaml index 5d20a2973..165d243f9 100644 --- a/manifests/ate-install/generated/ate.dev_actortemplates.yaml +++ b/manifests/ate-install/generated/ate.dev_actortemplates.yaml @@ -227,6 +227,74 @@ spec: type: object maxItems: 10 type: array + resources: + description: |- + Resources declares the compute resources for each actor of this template. + Unlike a pod, an actor is sized by its Limits: the sandbox is built to the + CPU/memory limits (cgroup caps, and for the micro-VM the VM's vCPU count and + memory), the scheduler only places the actor on a worker whose capacity is + >= these limits, and the limits are supplied to the sandbox over the actor + RPCs. Because the size is baked into snapshots, it is part of the immutable + spec. Requests and claims are not supported (actors are sized by limits only). + A zero or absent limit leaves the sandbox at the runtime default (unlimited + for gVisor, the kata config for the micro-VM). + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object sandboxClass: default: gvisor description: |- @@ -424,6 +492,17 @@ spec: rule: '(has(self.sandboxClass) && self.sandboxClass == ''microvm'') || !has(self.snapshotsConfig.onResume) || (has(self.snapshotsConfig.onResume.fromData) ? self.snapshotsConfig.onResume.fromData : ''ColdBoot'') != ''Golden''' + - message: spec.resources.requests is not supported; actors are sized + by spec.resources.limits only + rule: '!has(self.resources) || !has(self.resources.requests)' + - message: spec.resources.claims is not supported + rule: '!has(self.resources) || !has(self.resources.claims)' + - message: For sandboxClass 'microvm', spec.resources.limits.memory must + be at least 512Mi (256Mi VMM reserve + 256Mi guest minimum); below + this the VM cannot boot + rule: '!has(self.sandboxClass) || self.sandboxClass != ''microvm'' || + !has(self.resources) || !has(self.resources.limits) || !(''memory'' + in self.resources.limits) || !quantity(self.resources.limits[''memory'']).isLessThan(quantity(''512Mi''))' status: description: status is the observed state of ActorTemplate properties: diff --git a/pkg/api/v1alpha1/actortemplate_types.go b/pkg/api/v1alpha1/actortemplate_types.go index bda1346d1..1b659ee2e 100644 --- a/pkg/api/v1alpha1/actortemplate_types.go +++ b/pkg/api/v1alpha1/actortemplate_types.go @@ -15,6 +15,7 @@ package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -312,6 +313,17 @@ type SnapshotsConfig struct { // +kubebuilder:validation:XValidation:rule="!has(self.volumes) || self.volumes.all(v, has(self.containers) && self.containers.exists(c, has(c.volumeMounts) && c.volumeMounts.exists(vm, vm.name == v.name)))",message="All volumes defined in spec.volumes must be mounted by at least one container" // +kubebuilder:validation:XValidation:rule="!has(self.sandboxClass) || self.sandboxClass != 'microvm' || !has(self.volumes) || !self.volumes.exists(v, has(v.externalVolumeTemplate))",message="ExternalVolumes are not supported when sandboxClass is 'microvm'" // +kubebuilder:validation:XValidation:rule="(has(self.sandboxClass) && self.sandboxClass == 'microvm') || !has(self.snapshotsConfig.onResume) || (has(self.snapshotsConfig.onResume.fromData) ? self.snapshotsConfig.onResume.fromData : 'ColdBoot') != 'Golden'",message="onResume.fromData: Golden is not supported when sandboxClass is 'gvisor'" +// +kubebuilder:validation:XValidation:rule="!has(self.resources) || !has(self.resources.requests)",message="spec.resources.requests is not supported; actors are sized by spec.resources.limits only" +// +kubebuilder:validation:XValidation:rule="!has(self.resources) || !has(self.resources.claims)",message="spec.resources.claims is not supported" +// A micro-VM's guest RAM is the declared memory limit minus a fixed VMM reserve +// (256Mi, held back for cloud-hypervisor + virtiofsd); below a 256Mi guest minimum +// the VM cannot boot. Reject at admission any micro-VM memory limit under 512Mi +// (256Mi reserve + 256Mi guest minimum) so it fails at create time rather than at +// cold boot — a coarse pre-filter; the reserve-aware check in ateom (see +// cmd/ateom-microvm/run.go: resolveGuestMemMiB) stays authoritative. The 512Mi floor +// assumes the default reserve; deployments that raise --vmm-mem-reserve-mib rely on +// the runtime check. gVisor has no reserve, so this only applies to micro-VM. +// +kubebuilder:validation:XValidation:rule="!has(self.sandboxClass) || self.sandboxClass != 'microvm' || !has(self.resources) || !has(self.resources.limits) || !('memory' in self.resources.limits) || !quantity(self.resources.limits['memory']).isLessThan(quantity('512Mi'))",message="For sandboxClass 'microvm', spec.resources.limits.memory must be at least 512Mi (256Mi VMM reserve + 256Mi guest minimum); below this the VM cannot boot" type ActorTemplateSpec struct { // Containers is the workload definition. // @@ -358,6 +370,19 @@ type ActorTemplateSpec struct { // +optional // +kubebuilder:validation:MaxItems=32 Volumes []Volume `json:"volumes,omitempty"` + + // Resources declares the compute resources for each actor of this template. + // Unlike a pod, an actor is sized by its Limits: the sandbox is built to the + // CPU/memory limits (cgroup caps, and for the micro-VM the VM's vCPU count and + // memory), the scheduler only places the actor on a worker whose capacity is + // >= these limits, and the limits are supplied to the sandbox over the actor + // RPCs. Because the size is baked into snapshots, it is part of the immutable + // spec. Requests and claims are not supported (actors are sized by limits only). + // A zero or absent limit leaves the sandbox at the runtime default (unlimited + // for gVisor, the kata config for the micro-VM). + // + // +optional + Resources *corev1.ResourceRequirements `json:"resources,omitempty"` } // TODO: add validation diff --git a/pkg/api/v1alpha1/actortemplate_validation_test.go b/pkg/api/v1alpha1/actortemplate_validation_test.go index 33b54da29..7499323d7 100644 --- a/pkg/api/v1alpha1/actortemplate_validation_test.go +++ b/pkg/api/v1alpha1/actortemplate_validation_test.go @@ -413,6 +413,90 @@ func TestActorTemplateValidation(t *testing.T) { at.Spec.SandboxClass = SandboxClassMicroVM }, wantErr: false, + }, { + name: "microvm memory limit at the 512Mi floor is valid", + mutate: func(at *ActorTemplate) { + at.Spec.SandboxClass = SandboxClassMicroVM + at.Spec.Resources = &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("512Mi")}, + } + }, + wantErr: false, + }, { + name: "microvm memory limit above the floor is valid", + mutate: func(at *ActorTemplate) { + at.Spec.SandboxClass = SandboxClassMicroVM + at.Spec.Resources = &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1536Mi")}, + } + }, + wantErr: false, + }, { + name: "microvm memory limit below the floor is rejected", + mutate: func(at *ActorTemplate) { + at.Spec.SandboxClass = SandboxClassMicroVM + at.Spec.Resources = &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("256Mi")}, + } + }, + wantErr: true, + errMsg: "must be at least 512Mi", + }, { + name: "microvm memory limit just below the floor is rejected", + mutate: func(at *ActorTemplate) { + at.Spec.SandboxClass = SandboxClassMicroVM + at.Spec.Resources = &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("511Mi")}, + } + }, + wantErr: true, + errMsg: "must be at least 512Mi", + }, { + name: "microvm with no resources is valid (floor only applies to a set limit)", + mutate: func(at *ActorTemplate) { + at.Spec.SandboxClass = SandboxClassMicroVM + }, + wantErr: false, + }, { + name: "gvisor is exempt from the micro-VM memory floor", + mutate: func(at *ActorTemplate) { + at.Spec.SandboxClass = SandboxClassGvisor + at.Spec.Resources = &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("64Mi")}, + } + }, + wantErr: false, + }, { + name: "resources with requests is rejected", + mutate: func(at *ActorTemplate) { + at.Spec.Resources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2")}, + } + }, + wantErr: true, + errMsg: "spec.resources.requests is not supported", + }, { + name: "resources with claims is rejected", + mutate: func(at *ActorTemplate) { + at.Spec.Resources = &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2")}, + Claims: []corev1.ResourceClaim{{Name: "claim-1"}}, + } + }, + wantErr: true, + errMsg: "spec.resources.claims is not supported", + }, { + name: "resources with limits only is accepted", + mutate: func(at *ActorTemplate) { + at.Spec.Resources = &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + } + }, + wantErr: false, }, { name: "invalid SandboxClass", mutate: func(at *ActorTemplate) { diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index fb0b8841a..8b54a4259 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -106,6 +106,11 @@ func (in *ActorTemplateSpec) DeepCopyInto(out *ActorTemplateSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(corev1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ActorTemplateSpec. diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index 32ef71075..479a1065e 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -4530,8 +4530,10 @@ type Worker struct { SandboxClass string `protobuf:"bytes,9,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` Labels map[string]string `protobuf:"bytes,10,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` State Worker_State `protobuf:"varint,11,opt,name=state,proto3,enum=ateapi.Worker_State" json:"state,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // The compute capacity this worker can give an actor sandbox. + Capacity *WorkerCapacity `protobuf:"bytes,12,opt,name=capacity,proto3" json:"capacity,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Worker) Reset() { @@ -4641,6 +4643,71 @@ func (x *Worker) GetState() Worker_State { return Worker_STATE_UNSPECIFIED } +func (x *Worker) GetCapacity() *WorkerCapacity { + if x != nil { + return x.Capacity + } + return nil +} + +// WorkerCapacity is the worker pod's compute capacity available to host an +// actor sandbox, taken from the ateom container's resource limits. The +// scheduler only places an actor on a worker whose capacity is >= the actor's +// declared resource limits. An unset message, or a zero field within it, means +// "unknown/unset" for that dimension: treated as unconstrained so placement is +// not blocked (matching the pre-capacity behavior). +type WorkerCapacity struct { + state protoimpl.MessageState `protogen:"open.v1"` + CpuMilli int64 `protobuf:"varint,1,opt,name=cpu_milli,json=cpuMilli,proto3" json:"cpu_milli,omitempty"` // CPU capacity in millicores (1000 = one core). + MemoryBytes int64 `protobuf:"varint,2,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` // Memory capacity in bytes. + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WorkerCapacity) Reset() { + *x = WorkerCapacity{} + mi := &file_ateapi_proto_msgTypes[67] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WorkerCapacity) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WorkerCapacity) ProtoMessage() {} + +func (x *WorkerCapacity) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[67] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WorkerCapacity.ProtoReflect.Descriptor instead. +func (*WorkerCapacity) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{67} +} + +func (x *WorkerCapacity) GetCpuMilli() int64 { + if x != nil { + return x.CpuMilli + } + return 0 +} + +func (x *WorkerCapacity) GetMemoryBytes() int64 { + if x != nil { + return x.MemoryBytes + } + return 0 +} + type Assignment struct { state protoimpl.MessageState `protogen:"open.v1"` ActorTemplate *KubeNamespacedObjectRef `protobuf:"bytes,1,opt,name=actor_template,json=actorTemplate,proto3" json:"actor_template,omitempty"` @@ -4652,7 +4719,7 @@ type Assignment struct { func (x *Assignment) Reset() { *x = Assignment{} - mi := &file_ateapi_proto_msgTypes[67] + mi := &file_ateapi_proto_msgTypes[68] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4664,7 +4731,7 @@ func (x *Assignment) String() string { func (*Assignment) ProtoMessage() {} func (x *Assignment) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[67] + mi := &file_ateapi_proto_msgTypes[68] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4677,7 +4744,7 @@ func (x *Assignment) ProtoReflect() protoreflect.Message { // Deprecated: Use Assignment.ProtoReflect.Descriptor instead. func (*Assignment) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{67} + return file_ateapi_proto_rawDescGZIP(), []int{68} } func (x *Assignment) GetActorTemplate() *KubeNamespacedObjectRef { @@ -4711,7 +4778,7 @@ type KubeNamespacedObjectRef struct { func (x *KubeNamespacedObjectRef) Reset() { *x = KubeNamespacedObjectRef{} - mi := &file_ateapi_proto_msgTypes[68] + mi := &file_ateapi_proto_msgTypes[69] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4723,7 +4790,7 @@ func (x *KubeNamespacedObjectRef) String() string { func (*KubeNamespacedObjectRef) ProtoMessage() {} func (x *KubeNamespacedObjectRef) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[68] + mi := &file_ateapi_proto_msgTypes[69] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4736,7 +4803,7 @@ func (x *KubeNamespacedObjectRef) ProtoReflect() protoreflect.Message { // Deprecated: Use KubeNamespacedObjectRef.ProtoReflect.Descriptor instead. func (*KubeNamespacedObjectRef) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{68} + return file_ateapi_proto_rawDescGZIP(), []int{69} } func (x *KubeNamespacedObjectRef) GetNamespace() string { @@ -4761,7 +4828,7 @@ type DebugClearRequest struct { func (x *DebugClearRequest) Reset() { *x = DebugClearRequest{} - mi := &file_ateapi_proto_msgTypes[69] + mi := &file_ateapi_proto_msgTypes[70] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4773,7 +4840,7 @@ func (x *DebugClearRequest) String() string { func (*DebugClearRequest) ProtoMessage() {} func (x *DebugClearRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[69] + mi := &file_ateapi_proto_msgTypes[70] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4786,7 +4853,7 @@ func (x *DebugClearRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DebugClearRequest.ProtoReflect.Descriptor instead. func (*DebugClearRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{69} + return file_ateapi_proto_rawDescGZIP(), []int{70} } type DebugClearResponse struct { @@ -4797,7 +4864,7 @@ type DebugClearResponse struct { func (x *DebugClearResponse) Reset() { *x = DebugClearResponse{} - mi := &file_ateapi_proto_msgTypes[70] + mi := &file_ateapi_proto_msgTypes[71] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4809,7 +4876,7 @@ func (x *DebugClearResponse) String() string { func (*DebugClearResponse) ProtoMessage() {} func (x *DebugClearResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[70] + mi := &file_ateapi_proto_msgTypes[71] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4822,7 +4889,7 @@ func (x *DebugClearResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DebugClearResponse.ProtoReflect.Descriptor instead. func (*DebugClearResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{70} + return file_ateapi_proto_rawDescGZIP(), []int{71} } type MintJWTRequest struct { @@ -4837,7 +4904,7 @@ type MintJWTRequest struct { func (x *MintJWTRequest) Reset() { *x = MintJWTRequest{} - mi := &file_ateapi_proto_msgTypes[71] + mi := &file_ateapi_proto_msgTypes[72] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4849,7 +4916,7 @@ func (x *MintJWTRequest) String() string { func (*MintJWTRequest) ProtoMessage() {} func (x *MintJWTRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[71] + mi := &file_ateapi_proto_msgTypes[72] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4862,7 +4929,7 @@ func (x *MintJWTRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintJWTRequest.ProtoReflect.Descriptor instead. func (*MintJWTRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{71} + return file_ateapi_proto_rawDescGZIP(), []int{72} } func (x *MintJWTRequest) GetAudience() []string { @@ -4923,7 +4990,7 @@ type MintJWTResponse struct { func (x *MintJWTResponse) Reset() { *x = MintJWTResponse{} - mi := &file_ateapi_proto_msgTypes[72] + mi := &file_ateapi_proto_msgTypes[73] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4935,7 +5002,7 @@ func (x *MintJWTResponse) String() string { func (*MintJWTResponse) ProtoMessage() {} func (x *MintJWTResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[72] + mi := &file_ateapi_proto_msgTypes[73] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4948,7 +5015,7 @@ func (x *MintJWTResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintJWTResponse.ProtoReflect.Descriptor instead. func (*MintJWTResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{72} + return file_ateapi_proto_rawDescGZIP(), []int{73} } func (x *MintJWTResponse) GetActorJwt() string { @@ -4980,7 +5047,7 @@ type MintCertRequest struct { func (x *MintCertRequest) Reset() { *x = MintCertRequest{} - mi := &file_ateapi_proto_msgTypes[73] + mi := &file_ateapi_proto_msgTypes[74] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4992,7 +5059,7 @@ func (x *MintCertRequest) String() string { func (*MintCertRequest) ProtoMessage() {} func (x *MintCertRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[73] + mi := &file_ateapi_proto_msgTypes[74] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5005,7 +5072,7 @@ func (x *MintCertRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintCertRequest.ProtoReflect.Descriptor instead. func (*MintCertRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{73} + return file_ateapi_proto_rawDescGZIP(), []int{74} } func (x *MintCertRequest) GetWorkerNamespace() string { @@ -5062,7 +5129,7 @@ type MintCertResponse struct { func (x *MintCertResponse) Reset() { *x = MintCertResponse{} - mi := &file_ateapi_proto_msgTypes[74] + mi := &file_ateapi_proto_msgTypes[75] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5074,7 +5141,7 @@ func (x *MintCertResponse) String() string { func (*MintCertResponse) ProtoMessage() {} func (x *MintCertResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[74] + mi := &file_ateapi_proto_msgTypes[75] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5087,7 +5154,7 @@ func (x *MintCertResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintCertResponse.ProtoReflect.Descriptor instead. func (*MintCertResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{74} + return file_ateapi_proto_rawDescGZIP(), []int{75} } func (x *MintCertResponse) GetActorCertificates() [][]byte { @@ -5390,7 +5457,7 @@ const file_ateapi_proto_rawDesc = "" + "page_token\x18\x03 \x01(\tR\tpageToken\"c\n" + "\x12ListActorsResponse\x12%\n" + "\x06actors\x18\x01 \x03(\v2\r.ateapi.ActorR\x06actors\x12&\n" + - "\x0fnext_page_token\x18\x02 \x01(\tR\rnextPageToken\"\x9a\x04\n" + + "\x0fnext_page_token\x18\x02 \x01(\tR\rnextPageToken\"\xce\x04\n" + "\x06Worker\x12)\n" + "\x10worker_namespace\x18\x01 \x01(\tR\x0fworkerNamespace\x12\x1f\n" + "\vworker_pool\x18\x02 \x01(\tR\n" + @@ -5407,14 +5474,18 @@ const file_ateapi_proto_rawDesc = "" + "\rsandbox_class\x18\t \x01(\tR\fsandboxClass\x122\n" + "\x06labels\x18\n" + " \x03(\v2\x1a.ateapi.Worker.LabelsEntryR\x06labels\x12*\n" + - "\x05state\x18\v \x01(\x0e2\x14.ateapi.Worker.StateR\x05state\x1a9\n" + + "\x05state\x18\v \x01(\x0e2\x14.ateapi.Worker.StateR\x05state\x122\n" + + "\bcapacity\x18\f \x01(\v2\x16.ateapi.WorkerCapacityR\bcapacity\x1a9\n" + "\vLabelsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"D\n" + "\x05State\x12\x15\n" + "\x11STATE_UNSPECIFIED\x10\x00\x12\x10\n" + "\fSTATE_ACTIVE\x10\x01\x12\x12\n" + - "\x0eSTATE_DRAINING\x10\x02\"\x9a\x01\n" + + "\x0eSTATE_DRAINING\x10\x02\"P\n" + + "\x0eWorkerCapacity\x12\x1b\n" + + "\tcpu_milli\x18\x01 \x01(\x03R\bcpuMilli\x12!\n" + + "\fmemory_bytes\x18\x02 \x01(\x03R\vmemoryBytes\"\x9a\x01\n" + "\n" + "Assignment\x12F\n" + "\x0eactor_template\x18\x01 \x01(\v2\x1f.ateapi.KubeNamespacedObjectRefR\ractorTemplate\x12'\n" + @@ -5513,7 +5584,7 @@ func file_ateapi_proto_rawDescGZIP() []byte { } var file_ateapi_proto_enumTypes = make([]protoimpl.EnumInfo, 9) -var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 80) +var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 81) var file_ateapi_proto_goTypes = []any{ (SnapshotContentScope)(0), // 0: ateapi.SnapshotContentScope (ActorSnapshotTagScope)(0), // 1: ateapi.ActorSnapshotTagScope @@ -5591,29 +5662,30 @@ var file_ateapi_proto_goTypes = []any{ (*ListActorsRequest)(nil), // 73: ateapi.ListActorsRequest (*ListActorsResponse)(nil), // 74: ateapi.ListActorsResponse (*Worker)(nil), // 75: ateapi.Worker - (*Assignment)(nil), // 76: ateapi.Assignment - (*KubeNamespacedObjectRef)(nil), // 77: ateapi.KubeNamespacedObjectRef - (*DebugClearRequest)(nil), // 78: ateapi.DebugClearRequest - (*DebugClearResponse)(nil), // 79: ateapi.DebugClearResponse - (*MintJWTRequest)(nil), // 80: ateapi.MintJWTRequest - (*MintJWTResponse)(nil), // 81: ateapi.MintJWTResponse - (*MintCertRequest)(nil), // 82: ateapi.MintCertRequest - (*MintCertResponse)(nil), // 83: ateapi.MintCertResponse - nil, // 84: ateapi.Selector.MatchLabelsEntry - nil, // 85: ateapi.ExternalVolume.VolumeContextEntry - nil, // 86: ateapi.SandboxAssets.AssetsEntry - nil, // 87: ateapi.ArchAssets.FilesEntry - nil, // 88: ateapi.Worker.LabelsEntry - (*timestamppb.Timestamp)(nil), // 89: google.protobuf.Timestamp - (*fieldmaskpb.FieldMask)(nil), // 90: google.protobuf.FieldMask + (*WorkerCapacity)(nil), // 76: ateapi.WorkerCapacity + (*Assignment)(nil), // 77: ateapi.Assignment + (*KubeNamespacedObjectRef)(nil), // 78: ateapi.KubeNamespacedObjectRef + (*DebugClearRequest)(nil), // 79: ateapi.DebugClearRequest + (*DebugClearResponse)(nil), // 80: ateapi.DebugClearResponse + (*MintJWTRequest)(nil), // 81: ateapi.MintJWTRequest + (*MintJWTResponse)(nil), // 82: ateapi.MintJWTResponse + (*MintCertRequest)(nil), // 83: ateapi.MintCertRequest + (*MintCertResponse)(nil), // 84: ateapi.MintCertResponse + nil, // 85: ateapi.Selector.MatchLabelsEntry + nil, // 86: ateapi.ExternalVolume.VolumeContextEntry + nil, // 87: ateapi.SandboxAssets.AssetsEntry + nil, // 88: ateapi.ArchAssets.FilesEntry + nil, // 89: ateapi.Worker.LabelsEntry + (*timestamppb.Timestamp)(nil), // 90: google.protobuf.Timestamp + (*fieldmaskpb.FieldMask)(nil), // 91: google.protobuf.FieldMask } var file_ateapi_proto_depIdxs = []int32{ 0, // 0: ateapi.LocalSnapshotInfo.content_scope:type_name -> ateapi.SnapshotContentScope - 84, // 1: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry - 89, // 2: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp - 89, // 3: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp + 85, // 1: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry + 90, // 2: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp + 90, // 3: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp 5, // 4: ateapi.ExternalVolume.status:type_name -> ateapi.ExternalVolume.Status - 85, // 5: ateapi.ExternalVolume.volume_context:type_name -> ateapi.ExternalVolume.VolumeContextEntry + 86, // 5: ateapi.ExternalVolume.volume_context:type_name -> ateapi.ExternalVolume.VolumeContextEntry 11, // 6: ateapi.Actor.metadata:type_name -> ateapi.ResourceMetadata 18, // 7: ateapi.Actor.actor_template_version:type_name -> ateapi.ObjectRef 6, // 8: ateapi.Actor.status:type_name -> ateapi.Actor.Status @@ -5660,8 +5732,8 @@ var file_ateapi_proto_depIdxs = []int32{ 32, // 49: ateapi.Volume.durable_dir:type_name -> ateapi.DurableDirVolumeSource 33, // 50: ateapi.Volume.external_volume_template:type_name -> ateapi.ExternalVolumeTemplate 2, // 51: ateapi.SandboxAssets.sandbox_class:type_name -> ateapi.SandboxClass - 86, // 52: ateapi.SandboxAssets.assets:type_name -> ateapi.SandboxAssets.AssetsEntry - 87, // 53: ateapi.ArchAssets.files:type_name -> ateapi.ArchAssets.FilesEntry + 87, // 52: ateapi.SandboxAssets.assets:type_name -> ateapi.SandboxAssets.AssetsEntry + 88, // 53: ateapi.ArchAssets.files:type_name -> ateapi.ArchAssets.FilesEntry 17, // 54: ateapi.CreateAtespaceRequest.atespace:type_name -> ateapi.Atespace 18, // 55: ateapi.GetAtespaceRequest.atespace:type_name -> ateapi.ObjectRef 17, // 56: ateapi.ListAtespacesResponse.atespaces:type_name -> ateapi.Atespace @@ -5669,7 +5741,7 @@ var file_ateapi_proto_depIdxs = []int32{ 21, // 58: ateapi.CreateActorTemplateRequest.actor_template:type_name -> ateapi.ActorTemplate 18, // 59: ateapi.GetActorTemplateRequest.actor_template:type_name -> ateapi.ObjectRef 21, // 60: ateapi.UpdateActorTemplateRequest.actor_template:type_name -> ateapi.ActorTemplate - 90, // 61: ateapi.UpdateActorTemplateRequest.update_mask:type_name -> google.protobuf.FieldMask + 91, // 61: ateapi.UpdateActorTemplateRequest.update_mask:type_name -> google.protobuf.FieldMask 21, // 62: ateapi.ListActorTemplatesResponse.actor_templates:type_name -> ateapi.ActorTemplate 18, // 63: ateapi.DeleteActorTemplateRequest.actor_template:type_name -> ateapi.ObjectRef 22, // 64: ateapi.CreateActorTemplateVersionRequest.actor_template_version:type_name -> ateapi.ActorTemplateVersion @@ -5680,7 +5752,7 @@ var file_ateapi_proto_depIdxs = []int32{ 18, // 69: ateapi.GetActorRequest.actor:type_name -> ateapi.ObjectRef 13, // 70: ateapi.CreateActorRequest.actor:type_name -> ateapi.Actor 13, // 71: ateapi.UpdateActorRequest.actor:type_name -> ateapi.Actor - 90, // 72: ateapi.UpdateActorRequest.update_mask:type_name -> google.protobuf.FieldMask + 91, // 72: ateapi.UpdateActorRequest.update_mask:type_name -> google.protobuf.FieldMask 18, // 73: ateapi.SuspendActorRequest.actor:type_name -> ateapi.ObjectRef 13, // 74: ateapi.SuspendActorResponse.actor:type_name -> ateapi.Actor 18, // 75: ateapi.PauseActorRequest.actor:type_name -> ateapi.ObjectRef @@ -5693,85 +5765,86 @@ var file_ateapi_proto_depIdxs = []int32{ 15, // 82: ateapi.ListActorSnapshotsResponse.snapshots:type_name -> ateapi.ActorSnapshot 16, // 83: ateapi.CreateActorSnapshotTagRequest.actor_snapshot_tag:type_name -> ateapi.ActorSnapshotTag 16, // 84: ateapi.UpdateActorSnapshotTagRequest.tag:type_name -> ateapi.ActorSnapshotTag - 90, // 85: ateapi.UpdateActorSnapshotTagRequest.update_mask:type_name -> google.protobuf.FieldMask + 91, // 85: ateapi.UpdateActorSnapshotTagRequest.update_mask:type_name -> google.protobuf.FieldMask 18, // 86: ateapi.DeleteActorSnapshotTagRequest.tag:type_name -> ateapi.ObjectRef 75, // 87: ateapi.ListWorkersResponse.workers:type_name -> ateapi.Worker 13, // 88: ateapi.ListActorsResponse.actors:type_name -> ateapi.Actor - 76, // 89: ateapi.Worker.assignment:type_name -> ateapi.Assignment - 88, // 90: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry + 77, // 89: ateapi.Worker.assignment:type_name -> ateapi.Assignment + 89, // 90: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry 8, // 91: ateapi.Worker.state:type_name -> ateapi.Worker.State - 77, // 92: ateapi.Assignment.actor_template:type_name -> ateapi.KubeNamespacedObjectRef - 18, // 93: ateapi.Assignment.actor:type_name -> ateapi.ObjectRef - 4, // 94: ateapi.MintCertRequest.purpose:type_name -> ateapi.ActorCertificatePurpose - 36, // 95: ateapi.SandboxAssets.AssetsEntry.value:type_name -> ateapi.ArchAssets - 37, // 96: ateapi.ArchAssets.FilesEntry.value:type_name -> ateapi.AssetFile - 54, // 97: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest - 55, // 98: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest - 56, // 99: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest - 57, // 100: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest - 59, // 101: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest - 61, // 102: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest - 63, // 103: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest - 64, // 104: ateapi.Control.GetActorSnapshot:input_type -> ateapi.GetActorSnapshotRequest - 65, // 105: ateapi.Control.GetActorSnapshotTag:input_type -> ateapi.GetActorSnapshotTagRequest - 66, // 106: ateapi.Control.ListActorSnapshots:input_type -> ateapi.ListActorSnapshotsRequest - 68, // 107: ateapi.Control.CreateActorSnapshotTag:input_type -> ateapi.CreateActorSnapshotTagRequest - 69, // 108: ateapi.Control.UpdateActorSnapshotTag:input_type -> ateapi.UpdateActorSnapshotTagRequest - 70, // 109: ateapi.Control.DeleteActorSnapshotTag:input_type -> ateapi.DeleteActorSnapshotTagRequest - 71, // 110: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest - 73, // 111: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest - 38, // 112: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest - 39, // 113: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest - 40, // 114: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest - 42, // 115: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest - 43, // 116: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest - 44, // 117: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest - 45, // 118: ateapi.Control.UpdateActorTemplate:input_type -> ateapi.UpdateActorTemplateRequest - 46, // 119: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest - 48, // 120: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest - 49, // 121: ateapi.Control.CreateActorTemplateVersion:input_type -> ateapi.CreateActorTemplateVersionRequest - 50, // 122: ateapi.Control.GetActorTemplateVersion:input_type -> ateapi.GetActorTemplateVersionRequest - 51, // 123: ateapi.Control.ListActorTemplateVersions:input_type -> ateapi.ListActorTemplateVersionsRequest - 53, // 124: ateapi.Control.DeleteActorTemplateVersion:input_type -> ateapi.DeleteActorTemplateVersionRequest - 78, // 125: ateapi.Debug.DebugClear:input_type -> ateapi.DebugClearRequest - 80, // 126: ateapi.ActorIdentity.MintJWT:input_type -> ateapi.MintJWTRequest - 82, // 127: ateapi.ActorIdentity.MintCert:input_type -> ateapi.MintCertRequest - 13, // 128: ateapi.Control.GetActor:output_type -> ateapi.Actor - 13, // 129: ateapi.Control.CreateActor:output_type -> ateapi.Actor - 13, // 130: ateapi.Control.UpdateActor:output_type -> ateapi.Actor - 58, // 131: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse - 60, // 132: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse - 62, // 133: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse - 13, // 134: ateapi.Control.DeleteActor:output_type -> ateapi.Actor - 15, // 135: ateapi.Control.GetActorSnapshot:output_type -> ateapi.ActorSnapshot - 16, // 136: ateapi.Control.GetActorSnapshotTag:output_type -> ateapi.ActorSnapshotTag - 67, // 137: ateapi.Control.ListActorSnapshots:output_type -> ateapi.ListActorSnapshotsResponse - 16, // 138: ateapi.Control.CreateActorSnapshotTag:output_type -> ateapi.ActorSnapshotTag - 16, // 139: ateapi.Control.UpdateActorSnapshotTag:output_type -> ateapi.ActorSnapshotTag - 16, // 140: ateapi.Control.DeleteActorSnapshotTag:output_type -> ateapi.ActorSnapshotTag - 72, // 141: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse - 74, // 142: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse - 17, // 143: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace - 17, // 144: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace - 41, // 145: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse - 17, // 146: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace - 21, // 147: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate - 21, // 148: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate - 21, // 149: ateapi.Control.UpdateActorTemplate:output_type -> ateapi.ActorTemplate - 47, // 150: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse - 21, // 151: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate - 22, // 152: ateapi.Control.CreateActorTemplateVersion:output_type -> ateapi.ActorTemplateVersion - 22, // 153: ateapi.Control.GetActorTemplateVersion:output_type -> ateapi.ActorTemplateVersion - 52, // 154: ateapi.Control.ListActorTemplateVersions:output_type -> ateapi.ListActorTemplateVersionsResponse - 22, // 155: ateapi.Control.DeleteActorTemplateVersion:output_type -> ateapi.ActorTemplateVersion - 79, // 156: ateapi.Debug.DebugClear:output_type -> ateapi.DebugClearResponse - 81, // 157: ateapi.ActorIdentity.MintJWT:output_type -> ateapi.MintJWTResponse - 83, // 158: ateapi.ActorIdentity.MintCert:output_type -> ateapi.MintCertResponse - 128, // [128:159] is the sub-list for method output_type - 97, // [97:128] is the sub-list for method input_type - 97, // [97:97] is the sub-list for extension type_name - 97, // [97:97] is the sub-list for extension extendee - 0, // [0:97] is the sub-list for field type_name + 76, // 92: ateapi.Worker.capacity:type_name -> ateapi.WorkerCapacity + 78, // 93: ateapi.Assignment.actor_template:type_name -> ateapi.KubeNamespacedObjectRef + 18, // 94: ateapi.Assignment.actor:type_name -> ateapi.ObjectRef + 4, // 95: ateapi.MintCertRequest.purpose:type_name -> ateapi.ActorCertificatePurpose + 36, // 96: ateapi.SandboxAssets.AssetsEntry.value:type_name -> ateapi.ArchAssets + 37, // 97: ateapi.ArchAssets.FilesEntry.value:type_name -> ateapi.AssetFile + 54, // 98: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest + 55, // 99: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest + 56, // 100: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest + 57, // 101: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest + 59, // 102: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest + 61, // 103: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest + 63, // 104: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest + 64, // 105: ateapi.Control.GetActorSnapshot:input_type -> ateapi.GetActorSnapshotRequest + 65, // 106: ateapi.Control.GetActorSnapshotTag:input_type -> ateapi.GetActorSnapshotTagRequest + 66, // 107: ateapi.Control.ListActorSnapshots:input_type -> ateapi.ListActorSnapshotsRequest + 68, // 108: ateapi.Control.CreateActorSnapshotTag:input_type -> ateapi.CreateActorSnapshotTagRequest + 69, // 109: ateapi.Control.UpdateActorSnapshotTag:input_type -> ateapi.UpdateActorSnapshotTagRequest + 70, // 110: ateapi.Control.DeleteActorSnapshotTag:input_type -> ateapi.DeleteActorSnapshotTagRequest + 71, // 111: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest + 73, // 112: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest + 38, // 113: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest + 39, // 114: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest + 40, // 115: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest + 42, // 116: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest + 43, // 117: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest + 44, // 118: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest + 45, // 119: ateapi.Control.UpdateActorTemplate:input_type -> ateapi.UpdateActorTemplateRequest + 46, // 120: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest + 48, // 121: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest + 49, // 122: ateapi.Control.CreateActorTemplateVersion:input_type -> ateapi.CreateActorTemplateVersionRequest + 50, // 123: ateapi.Control.GetActorTemplateVersion:input_type -> ateapi.GetActorTemplateVersionRequest + 51, // 124: ateapi.Control.ListActorTemplateVersions:input_type -> ateapi.ListActorTemplateVersionsRequest + 53, // 125: ateapi.Control.DeleteActorTemplateVersion:input_type -> ateapi.DeleteActorTemplateVersionRequest + 79, // 126: ateapi.Debug.DebugClear:input_type -> ateapi.DebugClearRequest + 81, // 127: ateapi.ActorIdentity.MintJWT:input_type -> ateapi.MintJWTRequest + 83, // 128: ateapi.ActorIdentity.MintCert:input_type -> ateapi.MintCertRequest + 13, // 129: ateapi.Control.GetActor:output_type -> ateapi.Actor + 13, // 130: ateapi.Control.CreateActor:output_type -> ateapi.Actor + 13, // 131: ateapi.Control.UpdateActor:output_type -> ateapi.Actor + 58, // 132: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse + 60, // 133: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse + 62, // 134: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse + 13, // 135: ateapi.Control.DeleteActor:output_type -> ateapi.Actor + 15, // 136: ateapi.Control.GetActorSnapshot:output_type -> ateapi.ActorSnapshot + 16, // 137: ateapi.Control.GetActorSnapshotTag:output_type -> ateapi.ActorSnapshotTag + 67, // 138: ateapi.Control.ListActorSnapshots:output_type -> ateapi.ListActorSnapshotsResponse + 16, // 139: ateapi.Control.CreateActorSnapshotTag:output_type -> ateapi.ActorSnapshotTag + 16, // 140: ateapi.Control.UpdateActorSnapshotTag:output_type -> ateapi.ActorSnapshotTag + 16, // 141: ateapi.Control.DeleteActorSnapshotTag:output_type -> ateapi.ActorSnapshotTag + 72, // 142: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse + 74, // 143: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse + 17, // 144: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace + 17, // 145: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace + 41, // 146: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse + 17, // 147: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace + 21, // 148: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate + 21, // 149: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate + 21, // 150: ateapi.Control.UpdateActorTemplate:output_type -> ateapi.ActorTemplate + 47, // 151: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse + 21, // 152: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate + 22, // 153: ateapi.Control.CreateActorTemplateVersion:output_type -> ateapi.ActorTemplateVersion + 22, // 154: ateapi.Control.GetActorTemplateVersion:output_type -> ateapi.ActorTemplateVersion + 52, // 155: ateapi.Control.ListActorTemplateVersions:output_type -> ateapi.ListActorTemplateVersionsResponse + 22, // 156: ateapi.Control.DeleteActorTemplateVersion:output_type -> ateapi.ActorTemplateVersion + 80, // 157: ateapi.Debug.DebugClear:output_type -> ateapi.DebugClearResponse + 82, // 158: ateapi.ActorIdentity.MintJWT:output_type -> ateapi.MintJWTResponse + 84, // 159: ateapi.ActorIdentity.MintCert:output_type -> ateapi.MintCertResponse + 129, // [129:160] is the sub-list for method output_type + 98, // [98:129] is the sub-list for method input_type + 98, // [98:98] is the sub-list for extension type_name + 98, // [98:98] is the sub-list for extension extendee + 0, // [0:98] is the sub-list for field type_name } func init() { file_ateapi_proto_init() } @@ -5796,7 +5869,7 @@ func file_ateapi_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_ateapi_proto_rawDesc), len(file_ateapi_proto_rawDesc)), NumEnums: 9, - NumMessages: 80, + NumMessages: 81, NumExtensions: 0, NumServices: 3, }, diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index 366515607..05b635e5c 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -889,6 +889,20 @@ message Worker { STATE_DRAINING = 2; } State state = 11; + + // The compute capacity this worker can give an actor sandbox. + WorkerCapacity capacity = 12; +} + +// WorkerCapacity is the worker pod's compute capacity available to host an +// actor sandbox, taken from the ateom container's resource limits. The +// scheduler only places an actor on a worker whose capacity is >= the actor's +// declared resource limits. An unset message, or a zero field within it, means +// "unknown/unset" for that dimension: treated as unconstrained so placement is +// not blocked (matching the pre-capacity behavior). +message WorkerCapacity { + int64 cpu_milli = 1; // CPU capacity in millicores (1000 = one core). + int64 memory_bytes = 2; // Memory capacity in bytes. } message Assignment {