Exec works, needs cleanup

.gitignore

@@ -17,3 +17,6 @@ tmp/**
 
 # Cloud Hypervisor binaries (embedded at build time)
 lib/vmm/binaries/cloud-hypervisor/*/*/cloud-hypervisor
+cloud-hypervisor
+cloud-hypervisor/**
+lib/system/exec_agent/exec-agent

Makefile

@@ -87,16 +87,21 @@ ensure-ch-binaries:
 		$(MAKE) download-ch-binaries; \
 	fi
 
+# Build exec-agent (guest binary) into its own directory for embedding
+lib/system/exec_agent/exec-agent: lib/system/exec_agent/main.go
+	@echo "Building exec-agent..."
+	cd lib/system/exec_agent && CGO_ENABLED=0 go build -ldflags="-s -w" -o exec-agent .
+
 # Build the binary
-build: ensure-ch-binaries | $(BIN_DIR)
+build: ensure-ch-binaries lib/system/exec_agent/exec-agent | $(BIN_DIR)
 	go build -tags containers_image_openpgp -o $(BIN_DIR)/hypeman ./cmd/api
 
 # Run in development mode with hot reload
 dev: $(AIR)
 	$(AIR) -c .air.toml
 
 # Run tests
-test: ensure-ch-binaries
+test: ensure-ch-binaries lib/system/exec_agent/exec-agent
 	go test -tags containers_image_openpgp -v -timeout 30s ./...
 
 # Generate JWT token for testing

cmd/api/api/exec.go

@@ -70,7 +70,7 @@ func (s *ApiService) ExecHandler(w http.ResponseWriter, r *http.Request) {
 	wsConn := &wsReadWriter{ws: ws, ctx: ctx}
 
 	// Execute via vsock
-	exit, err := system.ExecIntoInstance(ctx, uint32(inst.VsockCID), system.ExecOptions{
+	exit, err := system.ExecIntoInstance(ctx, inst.VsockSocket, system.ExecOptions{
 		Command: command,
 		Stdin:   wsConn,
 		Stdout:  wsConn,

cmd/api/api/exec_test.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/onkernel/hypeman/lib/oapi"
+	"github.com/onkernel/hypeman/lib/paths"
 	"github.com/onkernel/hypeman/lib/system"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -24,17 +25,25 @@ func TestExecInstanceNonTTY(t *testing.T) {
 
 	svc := newTestService(t)
 
+	// Ensure system files (kernel and initrd) are available
+	t.Log("Ensuring system files...")
+	systemMgr := system.NewManager(paths.New(svc.Config.DataDir))
+	err := systemMgr.EnsureSystemFiles(ctx())
+	require.NoError(t, err)
+	t.Log("System files ready")
+
 	// First, create and wait for the image to be ready
-	t.Log("Creating alpine image...")
+	// Use nginx which has a proper long-running process
+	t.Log("Creating nginx:alpine image...")
 	imgResp, err := svc.CreateImage(ctx(), oapi.CreateImageRequestObject{
 		Body: &oapi.CreateImageRequest{
-			Name: "docker.io/library/alpine:latest",
+			Name: "docker.io/library/nginx:alpine",
 		},
 	})
 	require.NoError(t, err)
 	imgCreated, ok := imgResp.(oapi.CreateImage202JSONResponse)
 	require.True(t, ok, "expected 202 response")
-	assert.Equal(t, "docker.io/library/alpine:latest", imgCreated.Name)
+	assert.Equal(t, "docker.io/library/nginx:alpine", imgCreated.Name)
 
 	// Wait for image to be ready (poll with timeout)
 	t.Log("Waiting for image to be ready...")
@@ -49,7 +58,7 @@ func TestExecInstanceNonTTY(t *testing.T) {
 			t.Fatal("Timeout waiting for image to be ready")
 		case <-ticker.C:
 			imgResp, err := svc.GetImage(ctx(), oapi.GetImageRequestObject{
-				Name: "docker.io/library/alpine:latest",
+				Name: "docker.io/library/nginx:alpine",
 			})
 			require.NoError(t, err)
 
@@ -68,7 +77,7 @@ func TestExecInstanceNonTTY(t *testing.T) {
 	instResp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{
 		Body: &oapi.CreateInstanceRequest{
 			Name:  "exec-test",
-			Image: "docker.io/library/alpine:latest",
+			Image: "docker.io/library/nginx:alpine",
 		},
 	})
 	require.NoError(t, err)
@@ -91,6 +100,25 @@ func TestExecInstanceNonTTY(t *testing.T) {
 	require.NotEmpty(t, actualInst.VsockSocket, "vsock socket path should be set")
 	t.Logf("vsock CID: %d, socket: %s", actualInst.VsockCID, actualInst.VsockSocket)
 
+	// Capture console log on failure
+	t.Cleanup(func() {
+		if t.Failed() {
+			consolePath := paths.New(svc.Config.DataDir).InstanceConsoleLog(inst.Id)
+			if consoleData, err := os.ReadFile(consolePath); err == nil {
+				t.Logf("=== Console Log (Failure) ===")
+				t.Logf("%s", string(consoleData))
+				t.Logf("=== End Console Log ===")
+			}
+		}
+	})
+
+	// Check if vsock socket exists
+	if _, err := os.Stat(actualInst.VsockSocket); err != nil {
+		t.Logf("vsock socket does not exist: %v", err)
+	} else {
+		t.Logf("vsock socket exists: %s", actualInst.VsockSocket)
+	}
+
 	// Wait for exec agent to be ready (retry a few times)
 	var exit *system.ExitStatus
 	var stdout, stderr outputBuffer
@@ -102,7 +130,7 @@ func TestExecInstanceNonTTY(t *testing.T) {
 		stdout = outputBuffer{}
 		stderr = outputBuffer{}
 
-		exit, execErr = system.ExecIntoInstance(ctx(), uint32(actualInst.VsockCID), system.ExecOptions{
+		exit, execErr = system.ExecIntoInstance(ctx(), actualInst.VsockSocket, system.ExecOptions{
 			Command: []string{"/bin/sh", "-c", "whoami"},
 			Stdin:   nil,
 			Stdout:  &stdout,
@@ -128,13 +156,14 @@ func TestExecInstanceNonTTY(t *testing.T) {
 	t.Logf("Command output: %q", outStr)
 	require.Contains(t, outStr, "root", "whoami should return root user")
 
-	// Test another command to verify filesystem access
-	t.Log("Testing exec command: ls /usr/local/bin/exec-agent")
+	// Test another command to verify filesystem access and container context
+	// We should see /docker-entrypoint.sh which is standard in nginx:alpine image
+	t.Log("Testing exec command: ls /docker-entrypoint.sh")
 	stdout = outputBuffer{}
 	stderr = outputBuffer{}
 
-	exit, err = system.ExecIntoInstance(ctx(), uint32(actualInst.VsockCID), system.ExecOptions{
-		Command: []string{"/bin/sh", "-c", "ls -la /usr/local/bin/exec-agent"},
+	exit, err = system.ExecIntoInstance(ctx(), actualInst.VsockSocket, system.ExecOptions{
+		Command: []string{"/bin/sh", "-c", "ls -la /docker-entrypoint.sh"},
 		Stdin:   nil,
 		Stdout:  &stdout,
 		Stderr:  &stderr,
@@ -146,7 +175,7 @@ func TestExecInstanceNonTTY(t *testing.T) {
 
 	outStr = stdout.String()
 	t.Logf("ls output: %q", outStr)
-	require.Contains(t, outStr, "exec-agent", "should see exec-agent binary in /usr/local/bin")
+	require.Contains(t, outStr, "docker-entrypoint.sh", "should see docker-entrypoint.sh file")
 
 	// Cleanup
 	t.Log("Cleaning up instance...")

cmd/api/main.go

@@ -55,10 +55,9 @@ func run() error {
 		logger.Error("failed to ensure system files", "error", err)
 		os.Exit(1)
 	}
-	kernelVer, initrdVer := app.SystemManager.GetDefaultVersions()
+	kernelVer := app.SystemManager.GetDefaultKernelVersion()
 	logger.Info("System files ready",
-		"kernel", kernelVer,
-		"initrd", initrdVer)
+		"kernel", kernelVer)
 
 	// Create router
 	r := chi.NewRouter()

go.mod

@@ -4,13 +4,15 @@ go 1.25.4
 
 require (
 	github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500
+	github.com/creack/pty v1.1.24
 	github.com/distribution/reference v0.6.0
 	github.com/getkin/kin-openapi v0.133.0
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-chi/chi/v5 v5.2.3
 	github.com/golang-jwt/jwt/v5 v5.3.0
 	github.com/google/go-containerregistry v0.20.6
 	github.com/google/wire v0.7.0
+	github.com/gorilla/websocket v1.5.3
 	github.com/joho/godotenv v1.5.1
 	github.com/mdlayher/vsock v1.2.1
 	github.com/nrednav/cuid2 v1.1.0
@@ -22,6 +24,7 @@ require (
 	github.com/stretchr/testify v1.11.1
 	github.com/u-root/u-root v0.15.0
 	golang.org/x/sync v0.17.0
+	golang.org/x/term v0.37.0
 )
 
 require (
@@ -41,7 +44,6 @@ require (
 	github.com/go-test/deep v1.1.1 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
-	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
@@ -67,7 +69,7 @@ require (
 	github.com/woodsbury/decimal128 v1.3.0 // indirect
 	golang.org/x/crypto v0.41.0 // indirect
 	golang.org/x/net v0.42.0 // indirect
-	golang.org/x/sys v0.37.0 // indirect
+	golang.org/x/sys v0.38.0 // indirect
 	google.golang.org/protobuf v1.36.10 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -17,6 +17,8 @@ github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 h1:6lhrsTEnloDPXye
 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
 github.com/containerd/stargz-snapshotter/estargz v0.16.3 h1:7evrXtoh1mSbGj/pfRccTampEyKpjpOnS3CyiV1Ebr8=
 github.com/containerd/stargz-snapshotter/estargz v0.16.3/go.mod h1:uyr4BfYfOj3G9WBVE8cOlQmXAbPN9VEQpBBeJIuOipU=
+github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
+github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
 github.com/cyphar/filepath-securejoin v0.5.0 h1:hIAhkRBMQ8nIeuVwcAoymp7MY4oherZdAxD+m0u9zaw=
 github.com/cyphar/filepath-securejoin v0.5.0/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -190,8 +192,10 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
-golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
+golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
+golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

lib/instances/configdisk.go

@@ -89,6 +89,7 @@ func (m *manager) generateConfigScript(inst *Instance, imageInfo *images.Image)
 	}
 
 	// Generate script as a readable template block
+	// ENTRYPOINT and CMD contain shell-quoted arrays that will be eval'd in init
 	script := fmt.Sprintf(`#!/bin/sh
 # Generated config for instance: %s
 
@@ -134,17 +135,18 @@ func shellQuote(s string) string {
 }
 
 // shellQuoteArray quotes each element of an array for safe shell evaluation
-// Each element is single-quoted to preserve special characters like semicolons
+// Returns a string that when assigned to a variable and later eval'd, will be properly split
 func shellQuoteArray(arr []string) string {
 	if len(arr) == 0 {
-		return "\"\""
+		return ""
 	}
 
 	quoted := make([]string, len(arr))
 	for i, s := range arr {
 		quoted[i] = shellQuote(s)
 	}
 
+	// Join with spaces and return as-is (will be eval'd later in init script)
 	return strings.Join(quoted, " ")
 }
 

lib/instances/create.go

@@ -100,8 +100,8 @@ func (m *manager) createInstance(
 		req.Env = make(map[string]string)
 	}
 
-	// 6. Get default system versions
-	kernelVer, initrdVer := m.systemManager.GetDefaultVersions()
+	// 6. Get default kernel version
+	kernelVer := m.systemManager.GetDefaultKernelVersion()
 
 	// 7. Create instance metadata
 	stored := &StoredMetadata{
@@ -117,7 +117,6 @@ func (m *manager) createInstance(
 		StartedAt:     nil,
 		StoppedAt:     nil,
 		KernelVersion: string(kernelVer),
-		InitrdVersion: string(initrdVer),
 		CHVersion:     vmm.V49_0, // Use latest
 		SocketPath:    m.paths.InstanceSocket(id),
 		DataDir:       m.paths.InstanceDir(id),
@@ -293,9 +292,9 @@ func (m *manager) startAndBootVM(
 
 // buildVMConfig creates the Cloud Hypervisor VmConfig
 func (m *manager) buildVMConfig(inst *Instance, imageInfo *images.Image) (vmm.VmConfig, error) {
-	// Get versioned system file paths
+	// Get system file paths
 	kernelPath, _ := m.systemManager.GetKernelPath(system.KernelVersion(inst.KernelVersion))
-	initrdPath, _ := m.systemManager.GetInitrdPath(system.InitrdVersion(inst.InitrdVersion))
+	initrdPath, _ := m.systemManager.GetInitrdPath()
 
 	// Payload configuration (kernel + initramfs)
 	payload := vmm.PayloadConfig{

lib/instances/manager_test.go

@@ -177,7 +177,6 @@ func TestCreateAndDeleteInstance(t *testing.T) {
 	assert.Equal(t, StateRunning, inst.State)
 	assert.False(t, inst.HasSnapshot)
 	assert.NotEmpty(t, inst.KernelVersion)
-	assert.NotEmpty(t, inst.InitrdVersion)
 
 	// Verify directories exist
 	p := paths.New(tmpDir)