From 3dc7b7ba7ef36eb9e26489780f306d8cc23e5133 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 30 Jun 2026 01:53:39 -0700 Subject: [PATCH 1/3] fix(observability): harden self-host trace pipeline --- .../docs.self-hosting-troubleshooting.tsx | 34 ++++++++ docker-compose.yml | 11 +++ package.json | 1 + scripts/smoke-observability-traces.mjs | 80 +++++++++++++++++++ .../selfhost-observability-config.test.ts | 74 +++++++++++++++++ 5 files changed, 200 insertions(+) create mode 100755 scripts/smoke-observability-traces.mjs create mode 100644 test/unit/selfhost-observability-config.test.ts diff --git a/apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx b/apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx index b05f8b340e..190d11087d 100644 --- a/apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx +++ b/apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx @@ -121,6 +121,40 @@ rees_analyzer_config_invalid`} docker compose logs gittensory | grep selfhost_job_dead`} /> +

Grafana traces error or show no data

+

+ The trace path is app or smoke process → OTEL collector → Tempo → Grafana. Tempo is only + started by the observability profile, and app traces are only emitted when{" "} + OTEL_TRACES_EXPORTER includes otlp. +

+ + +

Readiness fails

setTimeout(resolve, ${JSON.stringify(pollIntervalMs)})); +} +throw new Error("tempo did not return smoke trace " + traceId + ": " + last.slice(0, 300)); +`; + + execFileSync( + "docker", + ["compose", "exec", "-T", composeService, "node", "-e", script], + { + stdio: "inherit", + }, + ); +} diff --git a/test/unit/selfhost-observability-config.test.ts b/test/unit/selfhost-observability-config.test.ts new file mode 100644 index 0000000000..105d3cc306 --- /dev/null +++ b/test/unit/selfhost-observability-config.test.ts @@ -0,0 +1,74 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; + +function readYaml(path: string): unknown { + return parse(readFileSync(join(process.cwd(), path), "utf8")); +} + +function record(value: unknown): Record { + expect(value).toBeTruthy(); + expect(typeof value).toBe("object"); + return value as Record; +} + +describe("self-host observability trace config", () => { + it("gates Grafana and the OTEL collector on Tempo readiness", () => { + const compose = record(readYaml("docker-compose.yml")); + const services = record(compose.services); + const tempo = record(services.tempo); + + expect(tempo.healthcheck?.test).toEqual([ + "CMD", + "wget", + "-qO-", + "http://127.0.0.1:3200/ready", + ]); + expect(tempo.healthcheck?.start_period).toBe("20s"); + expect(tempo.healthcheck?.retries).toBe(12); + expect(record(services.grafana).depends_on?.tempo).toEqual({ + condition: "service_healthy", + }); + expect(record(services["otel-collector"]).depends_on?.tempo).toEqual({ + condition: "service_healthy", + }); + }); + + it("keeps the collector, Tempo, and Grafana data source on the same trace path", () => { + const collector = record(readYaml("otel/otel-collector-config.yml")); + const tempo = record(readYaml("tempo/tempo.yaml")); + const datasource = record( + readYaml("grafana/provisioning/datasources/tempo.yml"), + ); + + expect(record(collector.exporters)["otlp/tempo"].endpoint).toBe( + "tempo:4317", + ); + expect( + record(record(collector.service).pipelines).traces.exporters, + ).toEqual(["otlp/tempo"]); + expect( + record(record(record(record(tempo.distributor).receivers).otlp).protocols) + .grpc.endpoint, + ).toBe("0.0.0.0:4317"); + expect( + record(record(record(record(tempo.distributor).receivers).otlp).protocols) + .http.endpoint, + ).toBe("0.0.0.0:4318"); + expect(record(record(tempo.storage).trace).backend).toBe("local"); + expect(record(datasource.datasources?.[0]).url).toBe("http://tempo:3200"); + }); + + it("ships an operator smoke probe that verifies collector to Tempo retrieval", () => { + const script = readFileSync( + join(process.cwd(), "scripts/smoke-observability-traces.mjs"), + "utf8", + ); + + expect(script).toContain("http://otel-collector:4318/v1/traces"); + expect(script).toContain("http://tempo:3200/api/traces/"); + expect(script).toContain("gittensory-selfhost-smoke"); + expect(script).toContain("selfhost.observability.smoke"); + }); +}); From 48549aadec66688e1b761d6a90c37b1194f7095b Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 30 Jun 2026 03:33:58 -0700 Subject: [PATCH 2/3] fix(observability): keep tempo dependencies profile-safe --- .env.example | 2 +- docker-compose.yml | 4 +--- .../selfhost-observability-config.test.ts | 20 ++++++++++++++----- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.env.example b/.env.example index 3d09e3ebe3..dc01743d6a 100644 --- a/.env.example +++ b/.env.example @@ -209,7 +209,7 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # The observability profile starts Prometheus (scrapes /metrics) + Alertmanager (alert rules in # prometheus/rules/, routing in alertmanager/alertmanager.yml — silent until you fill in a receiver) + # Loki + Promtail (ship every container's logs to Loki) + Grafana (dashboards for metrics AND logs). -# GRAFANA_ADMIN_PASSWORD=changeme # REQUIRED when using --profile observability; compose fails if unset +# GRAFANA_ADMIN_PASSWORD=changeme # change before exposing Grafana; defaults to changeme for local-only smoke tests # # Maintainer dashboards (in addition to the infra dashboard): # • "Reviews & PRs (maintainer)" — per-repo + combined PR/review analytics from a redacted reporting DB export. diff --git a/docker-compose.yml b/docker-compose.yml index 86a49c247d..7dc10ca258 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -289,8 +289,6 @@ services: condition: service_started loki: condition: service_started - tempo: - condition: service_healthy reporting-exporter: condition: service_healthy ports: @@ -302,7 +300,7 @@ services: # Maintainer dashboards query only the redacted reporting export, never the live app DB. - grafana-reporting-data:/reporting:ro environment: - GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env before using --profile observability} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-changeme} GF_USERS_ALLOW_SIGN_UP: "false" GF_INSTALL_PLUGINS: frser-sqlite-datasource,grafana-github-datasource # Read-only fine-grained PAT for the GitHub data source provisioning ($GITHUB_TOKEN expansion). From .env. diff --git a/test/unit/selfhost-observability-config.test.ts b/test/unit/selfhost-observability-config.test.ts index 105d3cc306..f17ffe2c70 100644 --- a/test/unit/selfhost-observability-config.test.ts +++ b/test/unit/selfhost-observability-config.test.ts @@ -14,10 +14,12 @@ function record(value: unknown): Record { } describe("self-host observability trace config", () => { - it("gates Grafana and the OTEL collector on Tempo readiness", () => { + it("gates Tempo consumers without breaking the default Compose profile", () => { const compose = record(readYaml("docker-compose.yml")); const services = record(compose.services); const tempo = record(services.tempo); + const grafana = record(services.grafana); + const collector = record(services["otel-collector"]); expect(tempo.healthcheck?.test).toEqual([ "CMD", @@ -27,12 +29,20 @@ describe("self-host observability trace config", () => { ]); expect(tempo.healthcheck?.start_period).toBe("20s"); expect(tempo.healthcheck?.retries).toBe(12); - expect(record(services.grafana).depends_on?.tempo).toEqual({ - condition: "service_healthy", - }); - expect(record(services["otel-collector"]).depends_on?.tempo).toEqual({ + expect(grafana.depends_on?.tempo).toBeUndefined(); + expect(collector.depends_on?.tempo).toEqual({ condition: "service_healthy", }); + expect(grafana.environment?.GF_SECURITY_ADMIN_PASSWORD).toBe( + "${GRAFANA_ADMIN_PASSWORD:-changeme}", + ); + + for (const [name, service] of Object.entries(services)) { + const serviceRecord = record(service); + if (!serviceRecord.depends_on?.tempo) continue; + expect(serviceRecord.profiles, name).toContain("observability"); + expect(tempo.profiles, "tempo").toContain("observability"); + } }); it("keeps the collector, Tempo, and Grafana data source on the same trace path", () => { From e66e5746e6185e6a2efeab1631bd298a5743086c Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 30 Jun 2026 03:43:46 -0700 Subject: [PATCH 3/3] fix(observability): fail closed for grafana password --- .env.example | 3 ++- docker-compose.yml | 13 ++++++++++++- test/unit/selfhost-observability-config.test.ts | 9 ++++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index dc01743d6a..85bd9b9831 100644 --- a/.env.example +++ b/.env.example @@ -209,7 +209,8 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # The observability profile starts Prometheus (scrapes /metrics) + Alertmanager (alert rules in # prometheus/rules/, routing in alertmanager/alertmanager.yml — silent until you fill in a receiver) + # Loki + Promtail (ship every container's logs to Loki) + Grafana (dashboards for metrics AND logs). -# GRAFANA_ADMIN_PASSWORD=changeme # change before exposing Grafana; defaults to changeme for local-only smoke tests +# GRAFANA_ADMIN_PASSWORD= # REQUIRED at runtime when using --profile observability; generate a strong value +# GRAFANA_LOCAL_SMOKE_PASSWORD= # optional local-only fallback for smoke tests; never expose Grafana with this # # Maintainer dashboards (in addition to the infra dashboard): # • "Reviews & PRs (maintainer)" — per-repo + combined PR/review analytics from a redacted reporting DB export. diff --git a/docker-compose.yml b/docker-compose.yml index 7dc10ca258..592183e0f4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -300,11 +300,22 @@ services: # Maintainer dashboards query only the redacted reporting export, never the live app DB. - grafana-reporting-data:/reporting:ro environment: - GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-changeme} + # Compose interpolates every service even when --profile observability is inactive, so the required-password + # check lives in entrypoint below. Runtime still fails closed unless the operator sets GRAFANA_ADMIN_PASSWORD. + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-${GRAFANA_LOCAL_SMOKE_PASSWORD:-}} GF_USERS_ALLOW_SIGN_UP: "false" GF_INSTALL_PLUGINS: frser-sqlite-datasource,grafana-github-datasource # Read-only fine-grained PAT for the GitHub data source provisioning ($GITHUB_TOKEN expansion). From .env. GITHUB_TOKEN: "${GITHUB_TOKEN:-}" + entrypoint: + - /bin/sh + - -ec + - | + if [ -z "$${GF_SECURITY_ADMIN_PASSWORD:-}" ]; then + echo >&2 "Set GRAFANA_ADMIN_PASSWORD in .env before using --profile observability, or set GRAFANA_LOCAL_SMOKE_PASSWORD for local-only smoke tests." + exit 1 + fi + exec /run.sh reporting-exporter: image: alpine:3.20 diff --git a/test/unit/selfhost-observability-config.test.ts b/test/unit/selfhost-observability-config.test.ts index f17ffe2c70..c720732e31 100644 --- a/test/unit/selfhost-observability-config.test.ts +++ b/test/unit/selfhost-observability-config.test.ts @@ -34,7 +34,14 @@ describe("self-host observability trace config", () => { condition: "service_healthy", }); expect(grafana.environment?.GF_SECURITY_ADMIN_PASSWORD).toBe( - "${GRAFANA_ADMIN_PASSWORD:-changeme}", + "${GRAFANA_ADMIN_PASSWORD:-${GRAFANA_LOCAL_SMOKE_PASSWORD:-}}", + ); + expect(JSON.stringify(grafana)).not.toContain("changeme"); + expect(grafana.entrypoint).toEqual( + expect.arrayContaining([ + expect.stringContaining("Set GRAFANA_ADMIN_PASSWORD"), + expect.stringContaining("exec /run.sh"), + ]), ); for (const [name, service] of Object.entries(services)) {