switch write to memory to --rootfs-writes=disk (new): file writes go to a folder on the host machine's disk. The VM reaches that folder through a "pass-through window" called virtio-fs — a

.ko.yaml

@@ -20,6 +20,7 @@ defaultPlatforms:
 
 baseImageOverrides:
   github.com/agent-substrate/substrate/demos/sandbox: alpine
+  github.com/agent-substrate/substrate/demos/counter: alpine
   # ateom-microvm needs glibc (for the fetched cloud-hypervisor binary) and mount/umount
   # (to bind the image into the virtiofsd shared dir) — both in debian:stable-slim but
   # not in the distroless static default.

cmd/ateom-microvm/checkpoint.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/ch"
 	"github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata"
+	"github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/third_party/kata/agentpb"
 	"github.com/agent-substrate/substrate/internal/ateompath"
 	"github.com/agent-substrate/substrate/internal/imagecache"
 	"github.com/agent-substrate/substrate/internal/proto/ateompb"
@@ -47,11 +48,13 @@ import (
 //   - FULL: the whole guest. ateom drives the CH REST api-socket: pause -> snapshot
 //     file://<CheckpointStateDir> (config.json + state.json + sparse memory-ranges)
 //     -> tear the VMM down. Each container's rootfs is overlay(virtio-fs RO lower +
-//     guest-tmpfs upper), so the writable upper lives in guest RAM and is captured by
-//     the memory snapshot — process memory and rootfs writes both persist across
-//     suspend/resume. The RO lower is reconstructed from the OCI image at restore, so
-//     nothing rootfs-related ships. Durable-dir volumes are host-backed rather than in
-//     guest RAM, so they ship alongside as a tar.
+//     writable upper). In the default memory mode the upper lives in guest RAM and is
+//     captured by the memory snapshot — process memory and rootfs writes both persist
+//     across suspend/resume. In disk mode (--rootfs-writes=disk) the upper is
+//     host-backed like the durable-dir volumes and ships alongside as its own tar
+//     (see rootfsupper.go). The RO lower is reconstructed from the OCI image at
+//     restore, so it never ships. Durable-dir volumes are host-backed under either
+//     mode, so they ship alongside as a tar.
 //   - DATA: the durable-dir volumes only, as that same tar. The guest is discarded, so
 //     the actor cold-starts on restore with its volumes re-materialized.
 //
@@ -105,6 +108,17 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
 		return nil, fmt.Errorf("while waiting for CH api-socket: %w", err)
 	}
 
+	var dDrop time.Duration
+	if scope == ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL && actorHasDiskUpper(actorUID) {
+		tDrop := time.Now()
+		if err := s.dropGuestCaches(ctx, ra, actorUID, req.GetSpec().GetContainers()); err != nil {
+			slog.WarnContext(ctx, "Failed to drop guest caches before pause", slog.Any("err", err))
+		} else {
+			dDrop = time.Since(tDrop)
+			slog.InfoContext(ctx, "Successfully dropped guest page caches", slog.Duration("duration", dDrop))
+		}
+	}
+
 	tPause := time.Now()
 	if err := client.Pause(ctx); err != nil {
 		return nil, fmt.Errorf("while pausing guest: %w", err)
@@ -143,6 +157,23 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
 		dDurable = time.Since(tDurable)
 	}
 
+	// Disk-backed rootfs uppers: host-backed like the durable volumes, so the
+	// memory snapshot no longer carries the rootfs writes — ship them as their
+	// own tar, taken while the guest is paused (write-through share, so every
+	// completed guest write is already on the host). Only a Full snapshot
+	// carries it: under Data the workload cold-starts on restore, discarding
+	// rootfs state in either mode. Detected from the host dir the disk-mode
+	// boot created (actorHasDiskUpper), not the current flag, so the snapshot
+	// always matches the guest's actual mounts.
+	var dUpper time.Duration
+	if scope == ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL && actorHasDiskUpper(actorUID) {
+		tUpper := time.Now()
+		if err := tarRootfsUpper(ctx, ateompath.RootfsUpperDir(actorUID), checkpointDir); err != nil {
+			return nil, err
+		}
+		dUpper = time.Since(tUpper)
+	}
+
 	// Report exactly the files we wrote so atelet ships precisely this snapshot: for
 	// Full, the CH snapshot (config.json + state.json + memory-ranges + base-id) plus
 	// any durable-dir tar; for Data, that tar alone.
@@ -165,11 +196,12 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
 
 	s.actorLogger.EmitLifecycleLog("Actor checkpointed", actorRef, actorUID, templateNS, templateName)
 	slog.InfoContext(ctx, "Actor checkpointed", slog.String("id", actorUID), slog.Any("snapshot_files", snapshotFiles),
-		slog.String("scope", scope.String()), slog.Duration("pause", dPause),
+		slog.String("scope", scope.String()), slog.Duration("drop_caches", dDrop), slog.Duration("pause", dPause),
 		slog.Duration("snapshot", dSnapshot),
-		// The durable-dir tar runs while the guest is paused, so its cost is part
-		// of the suspend latency and scales with the volume's contents.
-		slog.Duration("durable_dir", dDurable), slog.Duration("teardown", dTeardown))
+		// The durable-dir + rootfs-upper tars run while the guest is paused, so
+		// their cost is part of the suspend latency and scales with the contents.
+		slog.Duration("durable_dir", dDurable), slog.Duration("rootfs_upper", dUpper),
+		slog.Duration("teardown", dTeardown))
 	return &ateompb.CheckpointWorkloadResponse{SnapshotFiles: snapshotFiles}, nil
 }
 
@@ -219,9 +251,9 @@ func (s *AteomService) snapshotVMState(ctx context.Context, client *ch.Client, r
 			slog.String("id", actorUID), slog.Duration("merge", time.Since(tMerge)))
 	}
 
-	// Nothing rootfs-related ships: the overlay's writable upper is a guest tmpfs, so
-	// the actor's rootfs writes are already in the memory snapshot above, and the RO
-	// lower is reconstructed from the OCI image at restore (it never changes).
+	// The RO lower never ships (reconstructed from the OCI image at restore). The
+	// writable upper is in the memory snapshot above when it is a guest tmpfs; a
+	// disk-backed upper ships as its own tar from CheckpointWorkload instead.
 	return dSnapshot, nil
 }
 
@@ -270,15 +302,23 @@ func (s *AteomService) teardownActor(ctx context.Context, id string, ra *running
 			_, _ = ra.chCmd.Process.Wait()
 		}
 		// Kill the virtiofsds (after CH, their only client): the overlay RO lower's
-		// and, when the actor has durable-dir volumes, the writable share's.
-		for _, cmd := range []*exec.Cmd{ra.vfsdCmd, ra.durableVfsdCmd} {
+		// and, when present, the writable durable-dir and rootfs upper shares'.
+		for _, cmd := range []*exec.Cmd{ra.vfsdCmd, ra.durableVfsdCmd, ra.upperVfsdCmd} {
 			if cmd != nil && cmd.Process != nil {
 				_ = cmd.Process.Kill()
 				_, _ = cmd.Process.Wait()
 			}
 		}
 	}
 
+	// Remove the disk-backed rootfs upper dir (a no-op in memory mode): ateom
+	// owns it — atelet's actor-dir reset doesn't know it — and its absence is
+	// what marks the actor as not disk-backed (actorHasDiskUpper). Runs after
+	// the snapshot tar above, which is already on disk.
+	if err := os.RemoveAll(ateompath.RootfsUpperDir(id)); err != nil {
+		slog.WarnContext(ctx, "Failed to remove rootfs upper dir", slog.String("actorUID", id), slog.Any("err", err))
+	}
+
 	// Sweep any leftover per-sandbox host-side state + orphaned per-sandbox
 	// processes. This is ateom's own cleanup (process kill + unmount + rm).
 	kata.CleanupSandboxState(ctx, id)
@@ -290,3 +330,43 @@ func (s *AteomService) teardownActor(ctx context.Context, id string, ra *running
 		slog.WarnContext(ctx, "Failed to unmount bundle rootfs overlays", slog.String("actorUID", id), slog.Any("err", err))
 	}
 }
+
+// dropGuestCaches drops guest page caches using ExecProcess against the workload container.
+func (s *AteomService) dropGuestCaches(ctx context.Context, ra *runningActor, actorUID string, containers []*ateompb.Container) error {
+	var ac *kata.AgentClient
+	if ra != nil && ra.logAgent != nil {
+		ac = ra.logAgent
+	} else {
+		vsockPath := kata.VsockSocketPath(actorUID)
+		var err error
+		ac, err = dialAgentRetry(ctx, vsockPath, 5*time.Second)
+		if err != nil {
+			return fmt.Errorf("dialing kata-agent: %w", err)
+		}
+		defer ac.Close()
+	}
+
+	if len(containers) == 0 {
+		return fmt.Errorf("no containers configured for actor")
+	}
+
+	targetContainer := overlayWorkloadID(containers[0].GetName())
+
+	req := &agentpb.ExecProcessRequest{
+		ContainerId: targetContainer,
+		ExecId:      "drop-caches-" + actorUID,
+		Process: &agentpb.Process{
+			Args: []string{"/bin/sh", "-c", "sync && echo 3 > /proc/sys/vm/drop_caches"},
+			User: &agentpb.User{
+				UID: 0,
+				GID: 0,
+			},
+		},
+	}
+
+	if err := ac.ExecProcess(ctx, req); err != nil {
+		return fmt.Errorf("executing drop_caches in container %q: %w", targetContainer, err)
+	}
+
+	return nil
+}

cmd/ateom-microvm/internal/kata/agentclient.go

@@ -203,6 +203,15 @@ func (a *AgentClient) AddARPNeighbors(ctx context.Context, neighbors []*agentpb.
 	return nil
 }
 
+// ExecProcess executes a process inside a container or sandbox via the agent.
+// Mirrors grpc.AgentService/ExecProcess (returns google.protobuf.Empty).
+func (a *AgentClient) ExecProcess(ctx context.Context, req *agentpb.ExecProcessRequest) error {
+	if err := a.client.Call(ctx, "grpc.AgentService", "ExecProcess", req, &emptypb.Empty{}); err != nil {
+		return fmt.Errorf("agent ExecProcess: %w", err)
+	}
+	return nil
+}
+
 // ReadStdout reads up to max bytes from the container process's stdout. It is a
 // unary RPC (NOT a server stream): each call returns whatever bytes the agent has
 // buffered (up to max), so callers loop until it returns an error — the agent

cmd/ateom-microvm/internal/kata/overlay_linux.go

@@ -17,9 +17,11 @@
 package kata
 
 // Each container's rootfs is an overlay: its OCI image served read-only over virtio-fs
-// (the lower) plus a guest tmpfs (the writable upper). The upper is in guest RAM, so
-// rootfs writes ride along in the memory snapshot and persist across suspend/resume.
-// This file holds the overlay-specific helpers.
+// (the lower) plus a writable upper. By default the upper is a guest tmpfs — in guest
+// RAM, so rootfs writes ride along in the memory snapshot and persist across
+// suspend/resume. With disk-backed rootfs writes (--rootfs-writes=disk, see
+// cmd/ateom-microvm/rootfsupper.go) the upper instead lives on the ateUpper virtio-fs
+// share, backed by host disk. This file holds the overlay-specific helpers.
 
 import (
 	"context"
@@ -54,6 +56,16 @@ const (
 	// volume's contents live at <guestDurableDir>/<volumeName> and are bind-mounted
 	// from there into the containers that declare the volume.
 	guestDurableDir = "/run/ateom-durable"
+
+	// UpperFsTag is the virtio-fs tag for the actor's disk-backed rootfs upper
+	// share (--rootfs-writes=disk), served by a third virtiofsd from
+	// ateompath.RootfsUpperDir on the host.
+	UpperFsTag = "ateUpper"
+	// guestUpperDiskDir is where the agent mounts UpperFsTag in the guest; each
+	// container's overlay upper/work then live under <guestUpperDiskDir>/<cid>.
+	// Deliberately distinct from the tmpfs OverlayUpperBase prefix so the two
+	// modes can never alias.
+	guestUpperDiskDir = "/run/ateom-upper-disk"
 )
 
 // GuestDurableVolumeDir is the in-guest path holding one durable volume's
@@ -71,12 +83,19 @@ func SharedDir(id string) string {
 // VirtiofsdSocketPath is the vhost-user-fs socket CH connects to for the fs device.
 func VirtiofsdSocketPath(id string) string { return filepath.Join(VMDir(id), "virtiofsd.sock") }
 
-// OverlayUpperBase is the in-guest mount point for one container's overlay upper/work.
-// It lives under /run (tmpfs) so the upper's writes are in guest RAM and ride along in
-// the memory-only snapshot (rootfs writes persist). Keyed on the container id, which is
-// stable across the actor's restore lineage.
+// OverlayUpperBase is the in-guest mount point for one container's overlay upper/work
+// in the default (memory) mode. It lives under /run (tmpfs) so the upper's writes are
+// in guest RAM and ride along in the memory-only snapshot (rootfs writes persist).
+// Keyed on the container id, which is stable across the actor's restore lineage.
 func OverlayUpperBase(containerID string) string { return "/run/ateom-upper/" + containerID }
 
+// DiskUpperBase is the in-guest mount point for one container's overlay upper/work
+// when rootfs writes are disk-backed: a subdirectory of the ateUpper virtio-fs share,
+// so the upper's writes land on host disk (ateompath.RootfsUpperDir) instead of guest
+// RAM — the memory snapshot stays lean and the upper ships as a tar instead. Keyed on
+// the container id like OverlayUpperBase.
+func DiskUpperBase(containerID string) string { return guestUpperDiskDir + "/" + containerID }
+
 // GuestSharedRootfs is the in-guest path the kataShared mount exposes a container's
 // rootfs at. A carrier container with this as Root.Path makes the agent bind it to
 // /run/kata-containers/<cid>/rootfs — a stable per-container path the overlay then
@@ -91,6 +110,12 @@ type VirtiofsdOptions struct {
 	// Cache is virtiofsd's --cache mode. Empty defaults to "always", which is
 	// only correct for a strictly read-only share (see virtiofsdArgs).
 	Cache string
+	// Xattr enables xattr passthrough (--xattr). Required for a share hosting an
+	// overlayfs upper: overlay records whiteouts and opaque directories as
+	// trusted.overlay.* xattrs in the upper, and without passthrough the guest's
+	// overlay mount cannot round-trip them to the host (deletes of lower files
+	// would fail or silently un-delete across suspend/resume).
+	Xattr bool
 	Log   io.Writer
 }
 
@@ -106,14 +131,18 @@ func virtiofsdArgs(o VirtiofsdOptions) []string {
 		// side changes underneath the guest (e.g. contents restored from a snapshot).
 		cache = "always"
 	}
-	return []string{
+	args := []string{
 		"--socket-path=" + o.SocketPath,
 		"--shared-dir=" + o.SharedDir,
 		"--cache=" + cache,
 		"--thread-pool-size=1",
 		"--announce-submounts",
 		"--migration-mode", "find-paths",
 	}
+	if o.Xattr {
+		args = append(args, "--xattr")
+	}
+	return args
 }
 
 // StartVirtiofsd launches virtiofsd in find-paths migration mode serving o.SharedDir
@@ -193,7 +222,9 @@ func ReconstructSharedDirFromImage(ctx context.Context, bundleRootfs, restoreID,
 //
 // withDurableShare additionally mounts the writable durable-dir share, whose
 // per-volume subdirectories the containers bind-mount at their declared paths.
-func (a *AgentClient) CreateSandboxForActor(ctx context.Context, sandboxID, hostname string, withDurableShare bool) error {
+// withUpperShare additionally mounts the writable disk-backed rootfs upper share,
+// under whose mount each container's overlay upper/work live (DiskUpperBase).
+func (a *AgentClient) CreateSandboxForActor(ctx context.Context, sandboxID, hostname string, withDurableShare, withUpperShare bool) error {
 	storages := []*agentpb.Storage{{
 		Driver:     virtioFSDriver,
 		Source:     FsTag,
@@ -208,6 +239,14 @@ func (a *AgentClient) CreateSandboxForActor(ctx context.Context, sandboxID, host
 			MountPoint: guestDurableDir,
 		})
 	}
+	if withUpperShare {
+		storages = append(storages, &agentpb.Storage{
+			Driver:     virtioFSDriver,
+			Source:     UpperFsTag,
+			Fstype:     typeVirtioFS,
+			MountPoint: guestUpperDiskDir,
+		})
+	}
 	return a.CreateSandbox(ctx, &agentpb.CreateSandboxRequest{
 		Hostname:  hostname,
 		SandboxId: sandboxID,
@@ -239,9 +278,11 @@ func (a *AgentClient) CreateCarrier(ctx context.Context, cid string, spec *specs
 
 // StartOverlayWorkload creates + starts one container with an overlayfs rootfs:
 // lower = the carrier's resolved bind (/run/kata-containers/<cid>/rootfs from the RO
-// virtio-fs base), upper/work = <upperBase>/{fs,work} on a guest tmpfs so rootfs writes
-// land in guest RAM (captured by the memory-only snapshot → persist). The agent creates
-// the upper/work dirs (create_directory) before mounting the overlay.
+// virtio-fs base), upper/work = <upperBase>/{fs,work}. upperBase is either a guest
+// tmpfs (OverlayUpperBase: writes land in guest RAM, captured by the memory-only
+// snapshot) or a directory on the disk-backed ateUpper share (DiskUpperBase: writes
+// land on host disk, shipped as a tar at checkpoint). The agent creates the upper/work
+// dirs (create_directory) before mounting the overlay.
 func (a *AgentClient) StartOverlayWorkload(ctx context.Context, cid, workloadID, upperBase string, spec *specs.Spec) error {
 	const createDir = "io.katacontainers.volume.overlayfs.create_directory"
 	sharedBase := "/run/kata-containers/" + cid + "/rootfs"
@@ -251,6 +292,11 @@ func (a *AgentClient) StartOverlayWorkload(ctx context.Context, cid, workloadID,
 	upper := upperBase + "/fs"
 	work := upperBase + "/work"
 
+	options := []string{"lowerdir=" + lower, "upperdir=" + upper, "workdir=" + work}
+	if strings.HasPrefix(upperBase, guestUpperDiskDir) {
+		options = append(options, "index=off", "metacopy=off", "userxattr")
+	}
+
 	storages := []*agentpb.Storage{
 		{
 			Driver:     virtioFSDriver,
@@ -265,7 +311,7 @@ func (a *AgentClient) StartOverlayWorkload(ctx context.Context, cid, workloadID,
 			Fstype:        "overlay",
 			MountPoint:    ovlRoot,
 			DriverOptions: []string{createDir + "=" + upper, createDir + "=" + work},
-			Options:       []string{"lowerdir=" + lower, "upperdir=" + upper, "workdir=" + work},
+			Options:       options,
 		},
 	}
 	pbSpec := SpecToAgentPB(spec)

cmd/ateom-microvm/internal/kata/overlay_linux_test.go

@@ -26,6 +26,7 @@ func TestVirtiofsdArgs(t *testing.T) {
 		name      string
 		opts      VirtiofsdOptions
 		wantCache string
+		wantXattr bool
 	}{
 		{
 			name:      "RO lower defaults to cache=always",
@@ -41,13 +42,30 @@ func TestVirtiofsdArgs(t *testing.T) {
 			},
 			wantCache: "--cache=auto",
 		},
+		{
+			name: "rootfs upper share passes xattrs through",
+			opts: VirtiofsdOptions{
+				SocketPath: "/run/vm/virtiofsd-upper.sock",
+				SharedDir:  "/var/lib/ateom-gvisor/actors/uid/rootfs-upper",
+				Cache:      "auto",
+				Xattr:      true,
+			},
+			wantCache: "--cache=auto",
+			wantXattr: true,
+		},
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
 			args := virtiofsdArgs(tc.opts)
 			if !slices.Contains(args, tc.wantCache) {
 				t.Errorf("args %v do not contain %q", args, tc.wantCache)
 			}
+			// Overlay whiteouts are trusted.overlay.* xattrs in the upper; a share
+			// hosting an upper must pass them through, and the others must not pay
+			// the passthrough cost.
+			if gotXattr := slices.Contains(args, "--xattr"); gotXattr != tc.wantXattr {
+				t.Errorf("args %v: --xattr present = %v, want %v", args, gotXattr, tc.wantXattr)
+			}
 			for _, want := range []string{
 				"--socket-path=" + tc.opts.SocketPath,
 				"--shared-dir=" + tc.opts.SharedDir,

cmd/ateom-microvm/internal/kata/restore.go

@@ -35,3 +35,10 @@ func VsockSocketPath(id string) string { return filepath.Join(VMDir(id), "clh.so
 func DurableVirtiofsdSocketPath(id string) string {
 	return filepath.Join(VMDir(id), "virtiofsd-durable.sock")
 }
+
+// UpperVirtiofsdSocketPath is the vhost-user-fs socket for the actor's writable
+// disk-backed rootfs upper share (--rootfs-writes=disk), served by a third
+// virtiofsd alongside the RO lower's and the durable share's.
+func UpperVirtiofsdSocketPath(id string) string {
+	return filepath.Join(VMDir(id), "virtiofsd-upper.sock")
+}