diff --git a/agent/internal/collectors/runtime_stats.go b/agent/internal/collectors/runtime_stats.go new file mode 100644 index 0000000000..4dde265d76 --- /dev/null +++ b/agent/internal/collectors/runtime_stats.go @@ -0,0 +1,31 @@ +package collectors + +import "runtime" + +// RuntimeStats is a snapshot of the agent's own Go runtime memory state, +// reported on every heartbeat so fleet-wide agent memory leaks are visible +// from the server without shell access to the device (issue #2389). +type RuntimeStats struct { + HeapAllocBytes uint64 `json:"heapAllocBytes"` + HeapInuseBytes uint64 `json:"heapInuseBytes"` + HeapReleasedBytes uint64 `json:"heapReleasedBytes"` + SysBytes uint64 `json:"sysBytes"` + NumGC uint32 `json:"numGc"` + Goroutines int `json:"goroutines"` +} + +// CollectRuntimeStats reads the Go runtime's memory statistics for this +// process. runtime.ReadMemStats is a brief stop-the-world read (microseconds +// for a typical agent heap) — cheap enough to call on every heartbeat. +func CollectRuntimeStats() *RuntimeStats { + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + return &RuntimeStats{ + HeapAllocBytes: ms.HeapAlloc, + HeapInuseBytes: ms.HeapInuse, + HeapReleasedBytes: ms.HeapReleased, + SysBytes: ms.Sys, + NumGC: ms.NumGC, + Goroutines: runtime.NumGoroutine(), + } +} diff --git a/agent/internal/collectors/runtime_stats_test.go b/agent/internal/collectors/runtime_stats_test.go new file mode 100644 index 0000000000..7a89e84546 --- /dev/null +++ b/agent/internal/collectors/runtime_stats_test.go @@ -0,0 +1,54 @@ +package collectors + +import ( + "encoding/json" + "testing" +) + +func TestCollectRuntimeStats(t *testing.T) { + stats := CollectRuntimeStats() + if stats == nil { + t.Fatal("CollectRuntimeStats returned nil") + } + + // A running Go process always has a non-empty heap and at least the + // current goroutine. + if stats.HeapAllocBytes == 0 { + t.Error("HeapAllocBytes should be non-zero for a live process") + } + if stats.HeapInuseBytes == 0 { + t.Error("HeapInuseBytes should be non-zero for a live process") + } + if stats.SysBytes == 0 { + t.Error("SysBytes should be non-zero for a live process") + } + if stats.Goroutines < 1 { + t.Errorf("Goroutines = %d, want >= 1", stats.Goroutines) + } + // HeapInuse counts whole spans, so it can never be below HeapAlloc. + if stats.HeapInuseBytes < stats.HeapAllocBytes { + t.Errorf("HeapInuseBytes (%d) < HeapAllocBytes (%d)", stats.HeapInuseBytes, stats.HeapAllocBytes) + } +} + +func TestRuntimeStatsJSONShape(t *testing.T) { + // The wire field names are a contract with the API heartbeatSchema + // (apps/api/src/routes/agents/schemas.ts agentRuntime) — renaming a JSON + // key silently drops the gauge server-side. + data, err := json.Marshal(CollectRuntimeStats()) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var m map[string]any + if err := json.Unmarshal(data, &m); err != nil { + t.Fatalf("unmarshal: %v", err) + } + for _, key := range []string{ + "heapAllocBytes", "heapInuseBytes", "heapReleasedBytes", + "sysBytes", "numGc", "goroutines", + } { + if _, ok := m[key]; !ok { + t.Errorf("expected JSON key %q missing", key) + } + } +} diff --git a/agent/internal/heartbeat/handlers.go b/agent/internal/heartbeat/handlers.go index 72eb5e9f97..9d698cf948 100644 --- a/agent/internal/heartbeat/handlers.go +++ b/agent/internal/heartbeat/handlers.go @@ -96,6 +96,9 @@ var handlerRegistry = map[string]CommandHandler{ // Log shipping tools.CmdSetLogLevel: handleSetLogLevel, + // Runtime diagnostics — on-demand pprof capture (#2389) + tools.CmdCapturePprof: handleCapturePprof, + // Auto-update management tools.CmdSetAutoUpdate: handleSetAutoUpdate, } diff --git a/agent/internal/heartbeat/handlers_diag.go b/agent/internal/heartbeat/handlers_diag.go new file mode 100644 index 0000000000..6d8c295acf --- /dev/null +++ b/agent/internal/heartbeat/handlers_diag.go @@ -0,0 +1,108 @@ +package heartbeat + +// On-demand runtime diagnostics (#2389). capture_pprof captures heap and/or +// goroutine profiles in-process and returns them base64-encoded in the command +// result. There is deliberately NO pprof HTTP listener: the agent is a root +// daemon and must expose nothing reachable off-box, so the signed/queued +// command path is the only trigger. + +import ( + "bytes" + "encoding/base64" + "fmt" + "runtime" + "runtime/pprof" + "time" + + "github.com/breeze-rmm/agent/internal/collectors" + "github.com/breeze-rmm/agent/internal/remote/tools" +) + +// maxProfileBytes caps a single captured profile's raw size. Heap and +// goroutine profiles are sampled/compact — typically well under 1 MB even for +// large processes — and the server-side command result caps stdout at 5 MB, +// so 1 MiB raw per profile (~1.37 MiB base64) keeps the combined result +// comfortably inside that. Var (not const) so tests can exercise the cap +// without allocating a gigantic heap. +var maxProfileBytes = 1 << 20 + +func handleCapturePprof(_ *Heartbeat, cmd Command) tools.CommandResult { + // Strict payload validation: key absent → default "all"; key present but + // not a string → error (don't let a malformed payload silently force a + // GC + double capture the caller never asked for). + profile := "all" + if raw, ok := cmd.Payload["profile"]; ok { + s, isString := raw.(string) + if !isString { + return tools.CommandResult{ + Status: "failed", + Error: fmt.Sprintf("profile must be a string, got %T", raw), + } + } + profile = s + } + + var wantHeap, wantGoroutine bool + switch profile { + case "all": + wantHeap, wantGoroutine = true, true + case "heap": + wantHeap = true + case "goroutine": + wantGoroutine = true + default: + return tools.CommandResult{ + Status: "failed", + Error: fmt.Sprintf("invalid profile %q: must be heap, goroutine, or all", profile), + } + } + + result := map[string]any{ + "capturedAt": time.Now().UTC().Format(time.RFC3339), + // Snapshot of the runtime gauges at capture time, so the profile can + // be correlated with the heartbeat trend without a second command. + "runtime": collectors.CollectRuntimeStats(), + } + + if wantHeap { + // Force a GC first so the heap profile reflects live (in-use) objects + // rather than garbage awaiting collection — same effect as + // net/http/pprof's ?gc=1. + runtime.GC() + b64, size, err := capturePprofProfile("heap") + if err != nil { + return tools.CommandResult{Status: "failed", Error: err.Error()} + } + result["heapProfileBase64"] = b64 + result["heapProfileBytes"] = size + } + + if wantGoroutine { + b64, size, err := capturePprofProfile("goroutine") + if err != nil { + return tools.CommandResult{Status: "failed", Error: err.Error()} + } + result["goroutineProfileBase64"] = b64 + result["goroutineProfileBytes"] = size + } + + return tools.NewSuccessResult(result, 0) +} + +// capturePprofProfile writes the named runtime/pprof profile (debug=0 → +// gzip-compressed protobuf, the format `go tool pprof` consumes) and returns +// it base64-encoded along with its raw byte size. +func capturePprofProfile(name string) (string, int, error) { + p := pprof.Lookup(name) + if p == nil { + return "", 0, fmt.Errorf("profile %q not found", name) + } + var buf bytes.Buffer + if err := p.WriteTo(&buf, 0); err != nil { + return "", 0, fmt.Errorf("failed to write %s profile: %v", name, err) + } + if buf.Len() > maxProfileBytes { + return "", 0, fmt.Errorf("%s profile is %d bytes, exceeds the %d byte result cap", name, buf.Len(), maxProfileBytes) + } + return base64.StdEncoding.EncodeToString(buf.Bytes()), buf.Len(), nil +} diff --git a/agent/internal/heartbeat/handlers_diag_test.go b/agent/internal/heartbeat/handlers_diag_test.go new file mode 100644 index 0000000000..2f6b9ca48d --- /dev/null +++ b/agent/internal/heartbeat/handlers_diag_test.go @@ -0,0 +1,164 @@ +package heartbeat + +import ( + "encoding/base64" + "encoding/json" + "testing" + + "github.com/breeze-rmm/agent/internal/collectors" +) + +// decodeDiagResult parses the JSON payload NewSuccessResult marshals into +// Stdout. +func decodeDiagResult(t *testing.T, stdout string) map[string]any { + t.Helper() + var result map[string]any + if err := json.Unmarshal([]byte(stdout), &result); err != nil { + t.Fatalf("result stdout is not valid JSON: %v", err) + } + return result +} + +// assertValidPprofBlob base64-decodes the named field and checks it looks +// like a debug=0 runtime/pprof profile (gzip-compressed protobuf, magic +// bytes 0x1f 0x8b). +func assertValidPprofBlob(t *testing.T, result map[string]any, field string) { + t.Helper() + b64, ok := result[field].(string) + if !ok || b64 == "" { + t.Fatalf("%s missing or not a string", field) + } + raw, err := base64.StdEncoding.DecodeString(b64) + if err != nil { + t.Fatalf("%s is not valid base64: %v", field, err) + } + if len(raw) < 2 || raw[0] != 0x1f || raw[1] != 0x8b { + t.Fatalf("%s does not start with gzip magic bytes (got % x)", field, raw[:min(2, len(raw))]) + } +} + +func TestHandleCapturePprofDefaultCapturesBoth(t *testing.T) { + res := handleCapturePprof(nil, Command{ID: "c1", Type: "capture_pprof"}) + if res.Status != "completed" { + t.Fatalf("status = %q (error: %q), want completed", res.Status, res.Error) + } + result := decodeDiagResult(t, res.Stdout) + assertValidPprofBlob(t, result, "heapProfileBase64") + assertValidPprofBlob(t, result, "goroutineProfileBase64") + + if _, ok := result["capturedAt"].(string); !ok { + t.Error("capturedAt missing") + } + rt, ok := result["runtime"].(map[string]any) + if !ok { + t.Fatal("runtime stats snapshot missing") + } + if goroutines, _ := rt["goroutines"].(float64); goroutines < 1 { + t.Errorf("runtime.goroutines = %v, want >= 1", rt["goroutines"]) + } +} + +func TestHandleCapturePprofHeapOnly(t *testing.T) { + res := handleCapturePprof(nil, Command{ + ID: "c2", Type: "capture_pprof", + Payload: map[string]any{"profile": "heap"}, + }) + if res.Status != "completed" { + t.Fatalf("status = %q (error: %q), want completed", res.Status, res.Error) + } + result := decodeDiagResult(t, res.Stdout) + assertValidPprofBlob(t, result, "heapProfileBase64") + if _, present := result["goroutineProfileBase64"]; present { + t.Error("goroutineProfileBase64 should not be present for profile=heap") + } +} + +func TestHandleCapturePprofGoroutineOnly(t *testing.T) { + res := handleCapturePprof(nil, Command{ + ID: "c3", Type: "capture_pprof", + Payload: map[string]any{"profile": "goroutine"}, + }) + if res.Status != "completed" { + t.Fatalf("status = %q (error: %q), want completed", res.Status, res.Error) + } + result := decodeDiagResult(t, res.Stdout) + assertValidPprofBlob(t, result, "goroutineProfileBase64") + if _, present := result["heapProfileBase64"]; present { + t.Error("heapProfileBase64 should not be present for profile=goroutine") + } +} + +func TestHandleCapturePprofRejectsUnknownProfile(t *testing.T) { + res := handleCapturePprof(nil, Command{ + ID: "c4", Type: "capture_pprof", + Payload: map[string]any{"profile": "cpu"}, + }) + if res.Status != "failed" { + t.Fatalf("status = %q, want failed for unknown profile", res.Status) + } + if res.Error == "" { + t.Error("expected a descriptive error for an unknown profile") + } +} + +func TestHandleCapturePprofSizeCap(t *testing.T) { + orig := maxProfileBytes + maxProfileBytes = 1 // every real profile exceeds one byte + defer func() { maxProfileBytes = orig }() + + res := handleCapturePprof(nil, Command{ID: "c5", Type: "capture_pprof"}) + if res.Status != "failed" { + t.Fatalf("status = %q, want failed when profile exceeds size cap", res.Status) + } + if res.Error == "" { + t.Error("expected size-cap error message") + } +} + +func TestHandleCapturePprofRejectsNonStringProfile(t *testing.T) { + res := handleCapturePprof(nil, Command{ + ID: "c6", Type: "capture_pprof", + Payload: map[string]any{"profile": 123}, + }) + if res.Status != "failed" { + t.Fatalf("status = %q, want failed for non-string profile", res.Status) + } + if res.Error == "" { + t.Error("expected a descriptive error for a non-string profile") + } +} + +// TestHeartbeatPayloadAgentRuntimeWireKey pins the OUTER wire key of the +// runtime gauges. The API heartbeatSchema matches on the literal +// "agentRuntime" (apps/api/src/routes/agents/schemas.ts) with .optional(), so +// renaming the struct tag — or dropping the assignment in sendHeartbeat — +// would silently darken the gauges fleet-wide while every other test on both +// sides stayed green. +func TestHeartbeatPayloadAgentRuntimeWireKey(t *testing.T) { + withStats, err := json.Marshal(HeartbeatPayload{ + Status: "ok", + AgentRuntime: collectors.CollectRuntimeStats(), + }) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var m map[string]any + if err := json.Unmarshal(withStats, &m); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if _, ok := m["agentRuntime"]; !ok { + t.Error(`payload with AgentRuntime set must serialize an "agentRuntime" key`) + } + + withoutStats, err := json.Marshal(HeartbeatPayload{Status: "ok"}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + m = map[string]any{} + if err := json.Unmarshal(withoutStats, &m); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if _, ok := m["agentRuntime"]; ok { + t.Error("nil AgentRuntime must be omitted (omitempty) so old-server compat is preserved") + } +} diff --git a/agent/internal/heartbeat/handlers_test.go b/agent/internal/heartbeat/handlers_test.go index d455b13d3a..05b2714c94 100644 --- a/agent/internal/heartbeat/handlers_test.go +++ b/agent/internal/heartbeat/handlers_test.go @@ -91,6 +91,9 @@ var allCommandTypes = []string{ // handlers.go — log shipping tools.CmdSetLogLevel, + // handlers.go — runtime diagnostics (handlers_diag.go) + tools.CmdCapturePprof, + // handlers_autoupdate.go tools.CmdSetAutoUpdate, diff --git a/agent/internal/heartbeat/heartbeat.go b/agent/internal/heartbeat/heartbeat.go index 31fd03db7c..3aa5195fd2 100644 --- a/agent/internal/heartbeat/heartbeat.go +++ b/agent/internal/heartbeat/heartbeat.go @@ -100,6 +100,10 @@ type HeartbeatPayload struct { // OneDrive helper state (Phase 2). Nil until a config has been applied on a // Windows box — omitempty then drops the field entirely. OneDriveDeviceState *onedrivehelper.DeviceState `json:"onedriveDeviceState,omitempty"` + // Agent's own Go runtime memory gauges (#2389). Collected every heartbeat + // (runtime.ReadMemStats is microseconds) so fleet-wide agent memory leaks + // are visible from the server without shell access to the device. + AgentRuntime *collectors.RuntimeStats `json:"agentRuntime,omitempty"` } type DesktopAccessState struct { @@ -2804,6 +2808,9 @@ func (h *Heartbeat) sendHeartbeat() { // it or when the query failed — omitempty then drops the field. payload.Battery = h.hardwareCol.CollectBattery() + // Agent's own runtime memory gauges (#2389). + payload.AgentRuntime = collectors.CollectRuntimeStats() + // OneDrive helper state (Phase 2). Nil until a config has been applied on a // Windows box — omitempty then drops the field entirely. h.onedriveMu.Lock() diff --git a/agent/internal/remote/tools/types.go b/agent/internal/remote/tools/types.go index 5d5b5bd801..cd1546c364 100644 --- a/agent/internal/remote/tools/types.go +++ b/agent/internal/remote/tools/types.go @@ -176,6 +176,11 @@ const ( // Log shipping CmdSetLogLevel = "set_log_level" + // Runtime diagnostics — on-demand pprof capture (#2389). No listening + // socket: profiles are captured in-process and returned in the command + // result, so nothing is reachable off-box. + CmdCapturePprof = "capture_pprof" + // Dev push (fast dev binary update) // Auto-update management CmdSetAutoUpdate = "set_auto_update" diff --git a/apps/api/src/routes/agents/heartbeat.test.ts b/apps/api/src/routes/agents/heartbeat.test.ts index 5ceab6dc8e..b56fd31930 100644 --- a/apps/api/src/routes/agents/heartbeat.test.ts +++ b/apps/api/src/routes/agents/heartbeat.test.ts @@ -2270,3 +2270,131 @@ describe('POST /agents/:id/heartbeat — state-change audit (finding #10)', () = expect(fields).toEqual(expect.arrayContaining(['status', 'hostname', 'agentServerUrl'])); }); }); + +describe('POST /agents/:id/heartbeat — agentRuntime gauges (#2389)', () => { + beforeEach(() => { + vi.clearAllMocks(); + + // Device lookup → returns a row + selectMock.mockReturnValueOnce( + selectChainResolving([ + { + id: 'device-1', + orgId: 'org-1', + siteId: 'site-1', + hostname: 'host-1', + osType: 'linux', + osVersion: 'Ubuntu 22.04', + osBuild: null, + architecture: 'amd64', + agentVersion: '0.65.10', + deviceRole: 'server', + deviceRoleSource: 'auto', + agentTokenHash: 'hash', + tokenIssuedAt: new Date(), + }, + ]), + ); + + updateMock.mockReturnValue({ + set: vi.fn(() => ({ + where: vi.fn(() => whereResultWithReturning()), + })), + }); + + selectMock.mockReturnValue(selectChainResolving([])); + getActiveTrustKeysetMock.mockResolvedValue([]); + }); + + // Finds the deviceMetrics insert among all insert calls by its cpuPercent + // marker column, so an unrelated insert (audit, agent logs) can't be + // mistaken for it. + function findMetricsInsert(valuesSpy: ReturnType): Record | undefined { + return valuesSpy.mock.calls + .map((call) => call[0] as Record) + .find((v) => v && typeof v === 'object' && 'cpuPercent' in v); + } + + it('persists agentRuntime into device_metrics.custom_metrics', async () => { + const valuesSpy = vi.fn().mockResolvedValue(undefined); + insertMock.mockReturnValue({ values: valuesSpy }); + + const agentRuntime = { + heapAllocBytes: 12_345_678, + heapInuseBytes: 23_456_789, + heapReleasedBytes: 1_048_576, + sysBytes: 99_999_999, + numGc: 42, + goroutines: 87, + }; + + const resp = await buildApp().request('/agents/device-1/heartbeat', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ ...minimalHeartbeatBody, agentRuntime }), + }); + + expect(resp.status).toBe(200); + const metricsInsert = findMetricsInsert(valuesSpy); + expect(metricsInsert).toBeDefined(); + expect(metricsInsert?.customMetrics).toEqual({ agentRuntime }); + }); + + it('writes customMetrics: null when an old agent omits agentRuntime', async () => { + const valuesSpy = vi.fn().mockResolvedValue(undefined); + insertMock.mockReturnValue({ values: valuesSpy }); + + const resp = await buildApp().request('/agents/device-1/heartbeat', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(minimalHeartbeatBody), + }); + + expect(resp.status).toBe(200); + const metricsInsert = findMetricsInsert(valuesSpy); + expect(metricsInsert).toBeDefined(); + expect(metricsInsert?.customMetrics).toBeNull(); + }); + + // NOTE: schema-level tolerance (malformed agentRuntime dropped via .catch) + // is covered in schemas.heartbeatTolerance.test.ts — this route test mocks + // zValidator out, so the handler never sees schema-dropped fields. + + it('warns loudly (no metrics insert) when agentRuntime arrives without metrics', async () => { + // The gauges ride the device_metrics insert; when OS metrics collection + // failed there is no row to attach them to, and that drop must be + // observable (see #2389 review) — not silent. + const valuesSpy = vi.fn().mockResolvedValue(undefined); + insertMock.mockReturnValue({ values: valuesSpy }); + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + + try { + const { metrics: _omitted, ...noMetricsBody } = minimalHeartbeatBody; + const resp = await buildApp().request('/agents/device-1/heartbeat', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + ...noMetricsBody, + metricsAvailable: false, + agentRuntime: { + heapAllocBytes: 1, + heapInuseBytes: 2, + heapReleasedBytes: 3, + sysBytes: 4, + numGc: 5, + goroutines: 6, + }, + }), + }); + + expect(resp.status).toBe(200); + expect(findMetricsInsert(valuesSpy)).toBeUndefined(); + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining('agentRuntime received without metrics'), + expect.objectContaining({ deviceId: 'device-1', goroutines: 6 }), + ); + } finally { + warnSpy.mockRestore(); + } + }); +}); diff --git a/apps/api/src/routes/agents/heartbeat.ts b/apps/api/src/routes/agents/heartbeat.ts index d914267dd3..40ed9b4fe9 100644 --- a/apps/api/src/routes/agents/heartbeat.ts +++ b/apps/api/src/routes/agents/heartbeat.ts @@ -612,8 +612,22 @@ heartbeatRoutes.post('/:id/heartbeat', bodyLimit({ maxSize: 5 * 1024 * 1024, onE bandwidthInBps: data.metrics.bandwidthInBps != null ? BigInt(data.metrics.bandwidthInBps) : null, bandwidthOutBps: data.metrics.bandwidthOutBps != null ? BigInt(data.metrics.bandwidthOutBps) : null, interfaceStats: data.metrics.interfaceStats ?? null, - processCount: data.metrics.processCount + processCount: data.metrics.processCount, + // Agent's own Go runtime memory gauges (#2389) — jsonb sidecar, so no + // migration; null (not {}) when an old agent doesn't send them. + customMetrics: data.agentRuntime ? { agentRuntime: data.agentRuntime } : null }); + } else if (data.agentRuntime) { + // #2389 — the gauges ride the device_metrics insert, and that table's OS + // columns are NOT NULL, so a heartbeat whose OS metrics collection failed + // (metricsAvailable=false) has no row to attach them to. That is exactly + // the state a memory-sick agent is likely to be in, so the drop must be + // loud rather than indistinguishable from "old agent never sent gauges". + console.warn('[heartbeat] agentRuntime received without metrics — runtime gauges dropped', { + deviceId: device.id, + goroutines: data.agentRuntime.goroutines, + heapInuseBytes: data.agentRuntime.heapInuseBytes, + }); } if (data.ipHistoryUpdate) { diff --git a/apps/api/src/routes/agents/schemas.heartbeatTolerance.test.ts b/apps/api/src/routes/agents/schemas.heartbeatTolerance.test.ts index b341e68c49..d118efec9e 100644 --- a/apps/api/src/routes/agents/schemas.heartbeatTolerance.test.ts +++ b/apps/api/src/routes/agents/schemas.heartbeatTolerance.test.ts @@ -396,3 +396,61 @@ describe('heartbeatSchema — watchdogState .catch collapse (#1121)', () => { } }); }); + +describe('heartbeatSchema — agentRuntime gauges (#2389)', () => { + const minimal = { + status: 'ok' as const, + agentVersion: '0.95.0', + }; + + const validRuntime = { + heapAllocBytes: 12_345_678, + heapInuseBytes: 23_456_789, + heapReleasedBytes: 1_048_576, + sysBytes: 99_999_999, + numGc: 42, + goroutines: 87, + }; + + it('parses a full agentRuntime snapshot', () => { + const result = heartbeatSchema.safeParse({ ...minimal, agentRuntime: validRuntime }); + expect(result.success).toBe(true); + if (!result.success) return; + expect(result.data.agentRuntime).toEqual(validRuntime); + }); + + it('accepts gauge values above 2^53 (uint64 counters from a big process)', () => { + const result = heartbeatSchema.safeParse({ + ...minimal, + agentRuntime: { ...validRuntime, sysBytes: 2 ** 60 }, + }); + expect(result.success).toBe(true); + if (!result.success) return; + expect(result.data.agentRuntime?.sysBytes).toBe(2 ** 60); + }); + + it('drops the whole agentRuntime object on a negative gauge rather than rejecting', () => { + const result = heartbeatSchema.safeParse({ + ...minimal, + agentRuntime: { ...validRuntime, heapAllocBytes: -5 }, + }); + expect(result.success).toBe(true); + if (!result.success) return; + expect(result.data.agentRuntime).toBeUndefined(); + }); + + it('drops the whole agentRuntime object when a required gauge is missing', () => { + const { goroutines: _omitted, ...partial } = validRuntime; + const result = heartbeatSchema.safeParse({ ...minimal, agentRuntime: partial }); + expect(result.success).toBe(true); + if (!result.success) return; + expect(result.data.agentRuntime).toBeUndefined(); + }); + + it('heartbeat without agentRuntime (old agent) still parses', () => { + const result = heartbeatSchema.safeParse(minimal); + expect(result.success).toBe(true); + if (!result.success) return; + expect(result.data.agentRuntime).toBeUndefined(); + }); +}); diff --git a/apps/api/src/routes/agents/schemas.ts b/apps/api/src/routes/agents/schemas.ts index aa78d949fe..9a2871d841 100644 --- a/apps/api/src/routes/agents/schemas.ts +++ b/apps/api/src/routes/agents/schemas.ts @@ -198,6 +198,18 @@ export const heartbeatSchema = z.object({ timeRemainingMinutes: z.number().int().min(0).optional().catch(undefined), timeToFullMinutes: z.number().int().min(0).optional().catch(undefined), }).optional().catch(undefined), + // Agent's own Go runtime memory gauges (#2389). Informational — a bad value + // drops the whole object (.catch) rather than 400-ing the heartbeat. + // Persisted into device_metrics.custom_metrics so fleet-wide agent memory + // leaks are visible without shell access to the device. + agentRuntime: z.object({ + heapAllocBytes: uint64Counter, + heapInuseBytes: uint64Counter, + heapReleasedBytes: uint64Counter, + sysBytes: uint64Counter, + numGc: z.number().int().min(0), + goroutines: z.number().int().min(0), + }).optional().catch(undefined), role: z.enum(['agent', 'watchdog']).optional(), watchdogState: z.string().optional().catch(undefined), // Watchdog-only: 24h restart accounting for the main agent (#799 Layer B). diff --git a/apps/api/src/services/commandQueue.ts b/apps/api/src/services/commandQueue.ts index 17b23d9ade..74caa74c62 100644 --- a/apps/api/src/services/commandQueue.ts +++ b/apps/api/src/services/commandQueue.ts @@ -124,6 +124,11 @@ export const CommandTypes = { // Log shipping SET_LOG_LEVEL: 'set_log_level', + // Runtime diagnostics — on-demand pprof capture from the agent (#2389). + // Profiles are captured in-process and returned base64 in the command + // result; the agent never opens a listening socket for this. + CAPTURE_PPROF: 'capture_pprof', + // Screenshot (AI Vision) TAKE_SCREENSHOT: 'take_screenshot', diff --git a/apps/api/src/services/commandTimeouts.ts b/apps/api/src/services/commandTimeouts.ts index bb4b5b7e1f..4cf0c7b9dc 100644 --- a/apps/api/src/services/commandTimeouts.ts +++ b/apps/api/src/services/commandTimeouts.ts @@ -57,6 +57,7 @@ const SHORT_TIMEOUT_TYPES = new Set([ CommandTypes.TAKE_SCREENSHOT, CommandTypes.COMPUTER_ACTION, CommandTypes.SET_LOG_LEVEL, + CommandTypes.CAPTURE_PPROF, CommandTypes.PERIPHERAL_POLICY_SYNC, CommandTypes.COLLECT_BOOT_PERFORMANCE, CommandTypes.MANAGE_STARTUP_ITEM,