diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..a6bea8a --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,205 @@ +name: Deploy + +# Deployment pipeline (#226). +# +# push to main → build + push image, deploy to staging +# release published → build + push image, deploy to production +# tag v* → same as a published release +# repository_dispatch(deploy) → production, triggered by release.yml +# +# Every deployment goes through the GitHub Deployments API so status shows up +# on the commit and, for a merged PR, in that PR's checks. + +on: + push: + branches: [main] + tags: ["v*"] + release: + types: [published] + repository_dispatch: + types: [deploy] + workflow_dispatch: + inputs: + environment: + description: "Target environment" + required: true + default: staging + type: choice + options: + - staging + - production + +concurrency: + group: deploy-${{ github.ref }} + cancel-in-progress: false + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +permissions: + contents: read + packages: write + deployments: write + +jobs: + # ── Decide where this run is going ───────────────────────────────────────── + target: + name: Resolve target environment + runs-on: ubuntu-latest + outputs: + environment: ${{ steps.resolve.outputs.environment }} + steps: + - name: Resolve environment + id: resolve + run: | + case "${{ github.event_name }}" in + release|repository_dispatch) target=production ;; + workflow_dispatch) target="${{ github.event.inputs.environment }}" ;; + push) + if [[ "${{ github.ref }}" == refs/tags/v* ]]; then + target=production + else + target=staging + fi + ;; + *) target=staging ;; + esac + echo "environment=$target" >> "$GITHUB_OUTPUT" + echo "Deploying to $target" + + # ── Build the image and push it to GHCR ──────────────────────────────────── + build-and-push: + name: Build and push image + needs: target + runs-on: ubuntu-latest + outputs: + image: ${{ steps.meta.outputs.tags }} + digest: ${{ steps.build.outputs.digest }} + steps: + - uses: actions/checkout@v5 + + - name: Set up Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to ${{ env.REGISTRY }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Derive image tags + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha,format=long + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and push + id: build + uses: docker/build-push-action@v6 + with: + context: . + target: production + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Summarise + run: | + { + echo "## Image published" + echo "" + echo "| Field | Value |" + echo "|-------|-------|" + echo "| Environment | ${{ needs.target.outputs.environment }} |" + echo "| Digest | \`${{ steps.build.outputs.digest }}\` |" + echo "| Tags | \`${{ steps.meta.outputs.tags }}\` |" + } >> "$GITHUB_STEP_SUMMARY" + + # ── Roll the new image out ───────────────────────────────────────────────── + deploy: + name: Deploy to ${{ needs.target.outputs.environment }} + needs: [target, build-and-push] + runs-on: ubuntu-latest + environment: + name: ${{ needs.target.outputs.environment }} + url: ${{ steps.release.outputs.url }} + steps: + - uses: actions/checkout@v5 + + - name: Open GitHub deployment + id: start + uses: actions/github-script@v7 + with: + script: | + const deployment = await github.rest.repos.createDeployment({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: context.sha, + environment: '${{ needs.target.outputs.environment }}', + auto_merge: false, + required_contexts: [], + description: 'Automated deploy from ${{ github.workflow }}', + }); + core.setOutput('id', deployment.data.id); + await github.rest.repos.createDeploymentStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + deployment_id: deployment.data.id, + state: 'in_progress', + log_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + }); + + - name: Release image to ${{ needs.target.outputs.environment }} + id: release + env: + ENVIRONMENT: ${{ needs.target.outputs.environment }} + IMAGE_DIGEST: ${{ needs.build-and-push.outputs.digest }} + DEPLOY_HOOK_URL: ${{ secrets.DEPLOY_HOOK_URL }} + run: | + set -euo pipefail + IMAGE="${REGISTRY}/${IMAGE_NAME}@${IMAGE_DIGEST}" + echo "Releasing ${IMAGE} to ${ENVIRONMENT}" + + if [ -z "${DEPLOY_HOOK_URL:-}" ]; then + echo "::warning::DEPLOY_HOOK_URL is not configured for ${ENVIRONMENT};"\ + "the image is published but no host was told to pull it." + echo "url=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # The hook is whatever the hosting platform exposes (Render, Fly, + # Railway, a self-hosted webhook). It receives the exact digest so the + # environment runs the image this workflow just built. + curl --fail --silent --show-error --location \ + --max-time 120 \ + --request POST "${DEPLOY_HOOK_URL}" \ + --header "Content-Type: application/json" \ + --data "{\"image\":\"${IMAGE}\",\"environment\":\"${ENVIRONMENT}\"}" + + echo "url=${DEPLOY_HOOK_URL%%\?*}" >> "$GITHUB_OUTPUT" + + - name: Report deployment status + if: always() + uses: actions/github-script@v7 + with: + script: | + const success = '${{ job.status }}' === 'success'; + await github.rest.repos.createDeploymentStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + deployment_id: ${{ steps.start.outputs.id }}, + state: success ? 'success' : 'failure', + environment_url: '${{ steps.release.outputs.url }}' || undefined, + log_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + description: success ? 'Deployment succeeded' : 'Deployment failed', + }); diff --git a/Dockerfile b/Dockerfile index 12f2765..19b8c95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,12 @@ FROM node:22-alpine AS production ENV NODE_ENV=production +# Cap the V8 old-space heap below the 512MB container memory limit (#225). +# The headroom covers the Node binary, native buffers and the RPC client, so a +# runaway polling loop hits an OOM inside Node — with a JS stack trace — rather +# than being SIGKILLed by the kernel with no diagnostics. +ENV NODE_OPTIONS="--max-old-space-size=384" + WORKDIR /app COPY package*.json ./ diff --git a/docker-compose.yml b/docker-compose.yml index efb4e9c..f5d4baa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,12 +10,24 @@ services: environment: - NODE_ENV=production - REDIS_URL=redis://redis:6379 + # Keep the V8 heap ceiling under the container memory limit below (#225). + - NODE_OPTIONS=--max-old-space-size=384 depends_on: redis: condition: service_healthy volumes: - ./src:/app/src:ro restart: unless-stopped + # Resource limits (#225). Compose v2 honours deploy.resources outside swarm, + # so these apply to a plain `docker compose up`. + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + reservations: + cpus: "0.25" + memory: 256M healthcheck: test: ["CMD", "wget", "-qO-", "http://localhost:3000/health"] interval: 30s @@ -30,6 +42,15 @@ services: volumes: - redis_data:/data restart: unless-stopped + command: ["redis-server", "--maxmemory", "192mb", "--maxmemory-policy", "allkeys-lru"] + deploy: + resources: + limits: + cpus: "0.25" + memory: 256M + reservations: + cpus: "0.1" + memory: 64M healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 10s diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md new file mode 100644 index 0000000..c2fdba2 --- /dev/null +++ b/docs/DEPLOYMENT.md @@ -0,0 +1,156 @@ +# Deployment + +How the backend gets built, what it is allowed to consume at runtime, and how it +reaches staging and production. + +## Resource requirements + +The service is a single Node process: an Express API plus a `node-cron` polling +loop that talks to Stellar RPC. Both the polling loop and the RPC client hold +buffers that grow with the number of tracked projects, so the container declares +hard limits. Without them a leak or a runaway loop can consume the whole host +and take neighbouring services down with it. + +| Resource | Limit | Reservation | Where it is set | +| ------------ | -------- | ----------- | ---------------------------------- | +| Memory | 512 MB | 256 MB | `docker-compose.yml` | +| CPU | 0.5 core | 0.25 core | `docker-compose.yml` | +| V8 heap | 384 MB | n/a | `NODE_OPTIONS` in the `Dockerfile` | +| Redis memory | 256 MB | 64 MB | `docker-compose.yml` | +| Redis CPU | 0.25 | 0.1 | `docker-compose.yml` | + +### Why the heap ceiling is lower than the memory limit + +`--max-old-space-size=384` sits ~128 MB below the 512 MB container limit. That +gap covers the Node binary, the C++ heap, native buffers and thread stacks — +things V8 does not count against old space. + +The point is which failure you get. If V8 is allowed to grow past the container +limit, the kernel OOM killer sends SIGKILL and you get nothing: no stack, no log +line, just a restart. With the ceiling set below the limit, V8 throws +`JavaScript heap out of memory` first and you get a stack trace pointing at what +was allocating. + +Redis is capped independently with `--maxmemory 192mb` and an `allkeys-lru` +eviction policy, so it evicts rather than growing into its own container limit. + +### Running with the limits + +`deploy.resources` is honoured by Compose v2 outside swarm mode, so the normal +path already applies them: + +```bash +docker compose up -d +``` + +For a bare `docker run`, pass them explicitly: + +```bash +docker run --memory=512m --cpus=0.5 \ + -e NODE_OPTIONS=--max-old-space-size=384 \ + -p 3000:3000 --env-file .env \ + ghcr.io//backend:latest +``` + +Verify what the running container actually got: + +```bash +docker stats --no-stream +docker inspect --format '{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}' +``` + +### Tuning + +Raise the limits together, keeping roughly the same gap. Doubling memory to +1 GB means `--max-old-space-size=768`. Raising only the container limit wastes +the extra headroom, since V8 will not grow into it. Raising only the heap +ceiling reintroduces the SIGKILL failure mode. + +If the process is being killed at 512 MB under normal load, look at the polling +loop's per-cycle allocations before increasing the ceiling. + +## CI/CD pipeline + +Three workflows, each with a distinct job: + +| Workflow | Trigger | Does | +| ------------- | --------------------------------- | ----------------------------- | +| `ci.yml` | push to `main`, PRs to `main` | build, test, dependency audit | +| `release.yml` | push to `main`, manual dispatch | version, changelog, git tag | +| `deploy.yml` | push to `main`, release, tag `v*` | build image, push, deploy | + +### What triggers which environment + +| Event | Environment | +| -------------------------------------- | ----------- | +| Push to `main` (including a merged PR) | staging | +| Published GitHub release | production | +| Tag matching `v*` | production | +| `repository_dispatch` type `deploy` | production | +| Manual `workflow_dispatch` | your choice | + +`release.yml` already fires a `repository_dispatch` with `event_type=deploy` +after a manual release, so a manual release reaches production without a second +step. + +### Image registry + +Images go to GitHub Container Registry at +`ghcr.io//backend`, authenticated with the built-in `GITHUB_TOKEN` — no +extra registry secret needed. Tags produced per build: + +- `main` — the branch build +- `sha-` — immutable, one per commit +- `1.2.3`, `1.2` — on semver tags +- `latest` — default branch only + +Deployments reference the image **by digest**, not by tag, so the environment +runs exactly the image the workflow built even if a tag is later moved. + +### Deployment status in PRs + +`deploy.yml` opens a GitHub Deployment before rolling out and closes it with +`success` or `failure` afterwards. Because the deployment is attached to the +merge commit, the result appears on the merged PR and in the repository's +Environments view, with a link back to the workflow run. + +### Configuration + +| Secret / variable | Required | Purpose | +| ----------------- | -------- | ------------------------------------ | +| `GITHUB_TOKEN` | built-in | GHCR push and deployment status | +| `DEPLOY_HOOK_URL` | yes | Endpoint told to pull the new digest | + +`DEPLOY_HOOK_URL` is whatever the hosting platform exposes — a Render or Railway +deploy hook, a Fly webhook, or a self-hosted endpoint. It receives: + +```json +{ "image": "ghcr.io//backend@sha256:...", "environment": "staging" } +``` + +Set it per environment under **Settings → Environments → staging / production** +so staging and production can point at different hosts. + +If `DEPLOY_HOOK_URL` is unset the workflow still builds and publishes the image, +logs a warning, and marks the deployment successful — useful before hosting is +wired up. Set it once the target exists, or the pipeline will silently stop +short of an actual rollout. + +### Rollback + +Deployments are pinned to digests, so rolling back means pointing the hook at an +earlier one: + +```bash +docker buildx imagetools inspect ghcr.io//backend:latest +gh workflow run deploy.yml -f environment=production +``` + +To redeploy a specific past commit, re-run that commit's `deploy.yml` run from +the Actions tab. + +## Related + +- [SETUP.md](SETUP.md) — local development setup +- `Dockerfile` — build stages and the heap ceiling +- `docker-compose.yml` — resource limits and service wiring diff --git a/src/__tests__/admin-keypair-cache.test.ts b/src/__tests__/admin-keypair-cache.test.ts new file mode 100644 index 0000000..eb11d02 --- /dev/null +++ b/src/__tests__/admin-keypair-cache.test.ts @@ -0,0 +1,121 @@ +/** + * Tests for the admin keypair cache (#227). + * + * getAdminKeypair() used to re-derive the Ed25519 keypair from the secret on + * every call, which in the cron loop meant one derivation per project per + * cycle. It now derives once and reuses the result. + */ + +const mockConfig: Record = { + STELLAR_NETWORK: "testnet", + ADMIN_SECRET_KEY: "", + RPC_URL: "https://soroban-testnet.stellar.org", + DB_POOL_MIN: 2, + DB_POOL_MAX: 10, + DB_POOL_ACQUIRE_TIMEOUT_MS: 5000, + DB_POOL_HEALTH_CHECK_INTERVAL_MS: 30000, + RPC_BREAKER_FAILURE_THRESHOLD: 5, + RPC_BREAKER_RECOVERY_TIMEOUT_MS: 30000, + TX_MAX_RETRIES: 4, + TX_RETRY_BASE_DELAY_MS: 200, + TX_RETRY_MAX_DELAY_MS: 10000, +}; + +jest.mock("../config", () => ({ + get config() { + return mockConfig; + }, +})); + +jest.mock("@stellar/stellar-sdk", () => ({ + Keypair: { + // Fresh object per call so identity comparison proves the cache is used. + fromSecret: jest.fn((secret: string) => ({ publicKey: () => `PUB:${secret}` })), + random: jest.fn().mockReturnValue({ secret: () => "SRANDOM" }), + }, + rpc: { + Server: jest.fn(), + Api: { GetTransactionStatus: { NOT_FOUND: "NOT_FOUND", FAILED: "FAILED" } }, + }, + Networks: { + TESTNET: "Test SDF Network ; September 2015", + PUBLIC: "Public Global Stellar Network ; September 2015", + }, + TransactionBuilder: { fromXDR: jest.fn() }, + Account: jest.fn(), + xdr: { + LedgerKey: { account: jest.fn() }, + LedgerKeyAccount: jest.fn(), + }, +})); + +import { getAdminKeypair, resetAdminKeypairCache } from "../lib/stellar"; +import { Keypair } from "@stellar/stellar-sdk"; + +const fromSecret = Keypair.fromSecret as unknown as jest.Mock; + +describe("getAdminKeypair caching (#227)", () => { + beforeEach(() => { + resetAdminKeypairCache(); + fromSecret.mockClear(); + mockConfig.ADMIN_SECRET_KEY = "SSECRET1"; + }); + + it("derives the keypair only once across repeated calls", () => { + getAdminKeypair(); + getAdminKeypair(); + getAdminKeypair(); + + expect(fromSecret).toHaveBeenCalledTimes(1); + expect(fromSecret).toHaveBeenCalledWith("SSECRET1"); + }); + + it("returns the identical cached instance on subsequent calls", () => { + const first = getAdminKeypair(); + const second = getAdminKeypair(); + + expect(second).toBe(first); + }); + + it("stays cached across a simulated cron cycle over many projects", () => { + for (let projectId = 1; projectId <= 50; projectId++) { + getAdminKeypair(); + } + + expect(fromSecret).toHaveBeenCalledTimes(1); + }); + + it("throws the same error when ADMIN_SECRET_KEY is not set", () => { + mockConfig.ADMIN_SECRET_KEY = ""; + + expect(() => getAdminKeypair()).toThrow("ADMIN_SECRET_KEY not set"); + expect(fromSecret).not.toHaveBeenCalled(); + }); + + it("still throws on a missing secret even after a successful derivation", () => { + getAdminKeypair(); + expect(fromSecret).toHaveBeenCalledTimes(1); + + mockConfig.ADMIN_SECRET_KEY = ""; + expect(() => getAdminKeypair()).toThrow("ADMIN_SECRET_KEY not set"); + }); + + it("re-derives when the configured secret changes", () => { + const first = getAdminKeypair(); + + mockConfig.ADMIN_SECRET_KEY = "SSECRET2"; + const second = getAdminKeypair(); + + expect(fromSecret).toHaveBeenCalledTimes(2); + expect(second).not.toBe(first); + expect(second.publicKey()).toBe("PUB:SSECRET2"); + }); + + it("resetAdminKeypairCache forces the next call to derive again", () => { + getAdminKeypair(); + resetAdminKeypairCache(); + getAdminKeypair(); + + expect(fromSecret).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/__tests__/container-resources.test.ts b/src/__tests__/container-resources.test.ts new file mode 100644 index 0000000..893357f --- /dev/null +++ b/src/__tests__/container-resources.test.ts @@ -0,0 +1,144 @@ +/** + * Tests for container resource limits (#225). + * + * Verifies the memory and CPU ceilings, the V8 heap ceiling, and that the heap + * ceiling leaves headroom below the container memory limit. + */ + +import * as fs from "fs"; +import * as path from "path"; +import { parse } from "yaml"; + +const ROOT = path.resolve(__dirname, "../../"); + +function read(relativePath: string): string { + return fs.readFileSync(path.join(ROOT, relativePath), "utf8"); +} + +/** Convert a Compose memory string ("512M", "1g", "268435456") to bytes. */ +function toBytes(value: string | number): number { + if (typeof value === "number") return value; + const match = /^(\d+(?:\.\d+)?)\s*([kmgKMG])?[bB]?$/.exec(value.trim()); + if (!match) throw new Error(`Unparseable memory value: ${value}`); + const amount = Number(match[1]); + const unit = (match[2] || "").toLowerCase(); + const multiplier = unit === "g" ? 1024 ** 3 : unit === "m" ? 1024 ** 2 : unit === "k" ? 1024 : 1; + return amount * multiplier; +} + +const MB = 1024 * 1024; + +type ComposeService = { + environment?: string[]; + command?: string[]; + deploy?: { + resources?: { + limits?: { cpus?: string; memory?: string }; + reservations?: { cpus?: string; memory?: string }; + }; + }; +}; + +const compose = parse(read("docker-compose.yml")) as { + services: Record; +}; +const dockerfile = read("Dockerfile"); + +function limitsFor(service: string) { + const limits = compose.services[service]?.deploy?.resources?.limits; + if (!limits) throw new Error(`No deploy.resources.limits for service "${service}"`); + return limits; +} + +describe("container resource limits (#225)", () => { + describe("backend service", () => { + it("declares a memory limit", () => { + const { memory } = limitsFor("backend"); + expect(memory).toBeDefined(); + expect(toBytes(memory!)).toBeGreaterThan(0); + }); + + it("caps memory at 512MB", () => { + expect(toBytes(limitsFor("backend").memory!)).toBe(512 * MB); + }); + + it("declares a CPU limit of half a core", () => { + expect(Number(limitsFor("backend").cpus)).toBe(0.5); + }); + + it("reserves less than it limits", () => { + const resources = compose.services.backend.deploy!.resources!; + expect(toBytes(resources.reservations!.memory!)).toBeLessThan( + toBytes(resources.limits!.memory!), + ); + expect(Number(resources.reservations!.cpus)).toBeLessThan(Number(resources.limits!.cpus)); + }); + }); + + describe("Node.js heap ceiling", () => { + it("sets --max-old-space-size in the production image", () => { + const productionStage = dockerfile.slice(dockerfile.indexOf("AS production")); + expect(productionStage).toMatch(/NODE_OPTIONS=.*--max-old-space-size=\d+/); + }); + + it("sets it to 384MB", () => { + const match = /--max-old-space-size=(\d+)/.exec(dockerfile); + expect(match).not.toBeNull(); + expect(Number(match![1])).toBe(384); + }); + + it("does not set it in the build stage", () => { + const buildStage = dockerfile.slice(0, dockerfile.indexOf("AS production")); + expect(buildStage).not.toMatch(/max-old-space-size/); + }); + + it("leaves headroom below the container memory limit", () => { + const heapMb = Number(/--max-old-space-size=(\d+)/.exec(dockerfile)![1]); + const limitMb = toBytes(limitsFor("backend").memory!) / MB; + + // V8 old space does not account for the Node binary, native buffers or + // thread stacks, so the ceiling must sit meaningfully below the limit. + expect(heapMb).toBeLessThan(limitMb); + expect(limitMb - heapMb).toBeGreaterThanOrEqual(64); + }); + + it("passes the same ceiling through compose", () => { + const environment = compose.services.backend.environment ?? []; + expect(environment).toEqual( + expect.arrayContaining([expect.stringContaining("--max-old-space-size=384")]), + ); + }); + }); + + describe("redis service", () => { + it("declares memory and CPU limits", () => { + const limits = limitsFor("redis"); + expect(toBytes(limits.memory!)).toBeGreaterThan(0); + expect(Number(limits.cpus)).toBeGreaterThan(0); + }); + + it("caps its own memory below the container limit and evicts", () => { + const command = (compose.services.redis.command ?? []).join(" "); + expect(command).toMatch(/--maxmemory\s+\d+mb/); + expect(command).toMatch(/allkeys-lru/); + + const maxMemoryMb = Number(/--maxmemory (\d+)mb/.exec(command)![1]); + expect(maxMemoryMb * MB).toBeLessThan(toBytes(limitsFor("redis").memory!)); + }); + }); + + describe("documentation", () => { + const docPath = path.join(ROOT, "docs", "DEPLOYMENT.md"); + + it("deployment docs exist", () => { + expect(fs.existsSync(docPath)).toBe(true); + }); + + it("documents the resource requirements", () => { + const content = fs.readFileSync(docPath, "utf8"); + expect(content).toMatch(/512 ?MB/i); + expect(content).toMatch(/0\.5/); + expect(content).toMatch(/max-old-space-size=384/); + }); + }); +}); diff --git a/src/__tests__/deployment-pipeline.test.ts b/src/__tests__/deployment-pipeline.test.ts new file mode 100644 index 0000000..c7ab91b --- /dev/null +++ b/src/__tests__/deployment-pipeline.test.ts @@ -0,0 +1,193 @@ +/** + * Tests for the deployment workflow (#226). + * + * Checks that the pipeline builds and publishes an image on push to main, + * deploys to staging on merge, deploys to production on a release tag, and + * reports status back through the GitHub Deployments API. + */ + +import * as fs from "fs"; +import * as path from "path"; +import { parse } from "yaml"; + +const ROOT = path.resolve(__dirname, "../../"); +const WORKFLOW_PATH = path.join(ROOT, ".github", "workflows", "deploy.yml"); + +const raw = fs.readFileSync(WORKFLOW_PATH, "utf8"); + +type Job = { + name?: string; + needs?: string | string[]; + outputs?: Record; + environment?: unknown; + steps?: Array<{ name?: string; uses?: string; run?: string; with?: Record }>; +}; + +// `on` is parsed by the YAML spec as the boolean true, so read it off the +// raw record rather than by name. +const workflow = parse(raw) as Record & { + jobs: Record; + env?: Record; + permissions?: Record; +}; + +const triggers = (workflow.on ?? workflow[true as unknown as string]) as Record; + +function job(name: string): Job { + const found = workflow.jobs[name]; + if (!found) throw new Error(`Workflow has no job "${name}". Jobs: ${Object.keys(workflow.jobs)}`); + return found; +} + +function stepsOf(name: string): string { + return JSON.stringify(job(name).steps ?? []); +} + +describe("deployment workflow (#226)", () => { + it("deploy.yml exists and parses", () => { + expect(fs.existsSync(WORKFLOW_PATH)).toBe(true); + expect(workflow.jobs).toBeDefined(); + }); + + describe("triggers", () => { + it("runs on push to main", () => { + const push = triggers.push as { branches?: string[]; tags?: string[] }; + expect(push.branches).toContain("main"); + }); + + it("runs on version tags", () => { + const push = triggers.push as { tags?: string[] }; + expect(push.tags).toEqual(expect.arrayContaining([expect.stringMatching(/^v/)])); + }); + + it("runs when a release is published", () => { + const release = triggers.release as { types?: string[] }; + expect(release.types).toContain("published"); + }); + + it("accepts the repository_dispatch fired by release.yml", () => { + const dispatch = triggers.repository_dispatch as { types?: string[] }; + expect(dispatch.types).toContain("deploy"); + + // release.yml sends event_type=deploy; the two must stay in step. + const release = fs.readFileSync(path.join(ROOT, ".github/workflows/release.yml"), "utf8"); + expect(release).toMatch(/event_type=deploy/); + }); + + it("can be run manually against either environment", () => { + const manual = triggers.workflow_dispatch as { + inputs?: { environment?: { options?: string[] } }; + }; + expect(manual.inputs?.environment?.options).toEqual( + expect.arrayContaining(["staging", "production"]), + ); + }); + }); + + describe("environment routing", () => { + const resolve = stepsOf("target"); + + it("routes a plain push to main at staging", () => { + expect(resolve).toMatch(/target=staging/); + }); + + it("routes releases and tags at production", () => { + expect(resolve).toMatch(/release\|repository_dispatch\).*target=production/); + expect(resolve).toMatch(/refs\/tags\/v\*/); + }); + + it("exposes the resolved environment to later jobs", () => { + expect(job("target").outputs?.environment).toBeDefined(); + }); + }); + + describe("image build and registry push", () => { + const build = stepsOf("build-and-push"); + + it("publishes to a container registry", () => { + expect(workflow.env?.REGISTRY).toBe("ghcr.io"); + expect(build).toMatch(/docker\/login-action/); + }); + + it("authenticates with the built-in token", () => { + expect(build).toMatch(/secrets\.GITHUB_TOKEN/); + expect(workflow.permissions?.packages).toBe("write"); + }); + + it("builds the production stage and pushes it", () => { + expect(build).toMatch(/docker\/build-push-action/); + expect(build).toMatch(/"target":\s*"production"/); + expect(build).toMatch(/"push":\s*true/); + }); + + it("tags images by commit sha as well as by branch", () => { + expect(build).toMatch(/type=sha/); + expect(build).toMatch(/type=ref,event=branch/); + expect(build).toMatch(/type=semver/); + }); + + it("exposes the image digest for the deploy job", () => { + expect(job("build-and-push").outputs?.digest).toBeDefined(); + }); + }); + + describe("deploy job", () => { + const deploy = stepsOf("deploy"); + + it("waits for the image before deploying", () => { + expect(job("deploy").needs).toEqual(expect.arrayContaining(["build-and-push"])); + }); + + it("targets the resolved environment", () => { + expect(JSON.stringify(job("deploy").environment)).toMatch(/target\.outputs\.environment/); + }); + + it("deploys the image by digest rather than by mutable tag", () => { + expect(deploy).toMatch(/IMAGE_DIGEST/); + expect(deploy).toMatch(/\$\{IMAGE_NAME\}@\$\{IMAGE_DIGEST\}/); + }); + + it("reads the deploy target from a secret", () => { + expect(deploy).toMatch(/secrets\.DEPLOY_HOOK_URL/); + }); + }); + + describe("status reporting", () => { + const deploy = stepsOf("deploy"); + + it("has permission to write deployments", () => { + expect(workflow.permissions?.deployments).toBe("write"); + }); + + it("opens a deployment and marks it in_progress", () => { + expect(deploy).toMatch(/createDeployment/); + expect(deploy).toMatch(/in_progress/); + }); + + it("always closes the deployment with a final state", () => { + expect(deploy).toMatch(/createDeploymentStatus/); + expect(deploy).toMatch(/'success' : 'failure'/); + expect(deploy).toMatch(/"if":\s*"always\(\)"/); + }); + + it("links the deployment back to the workflow run", () => { + expect(deploy).toMatch(/log_url/); + expect(deploy).toMatch(/context\.runId/); + }); + }); + + describe("safety", () => { + it("does not cancel a deployment already in flight", () => { + const concurrency = workflow.concurrency as { "cancel-in-progress"?: boolean }; + expect(concurrency["cancel-in-progress"]).toBe(false); + }); + + it("is documented", () => { + const docs = fs.readFileSync(path.join(ROOT, "docs", "DEPLOYMENT.md"), "utf8"); + expect(docs).toMatch(/DEPLOY_HOOK_URL/); + expect(docs).toMatch(/ghcr\.io/); + expect(docs).toMatch(/staging/); + expect(docs).toMatch(/production/); + }); + }); +}); diff --git a/src/__tests__/strict-type-guards.test.ts b/src/__tests__/strict-type-guards.test.ts new file mode 100644 index 0000000..936a2b0 --- /dev/null +++ b/src/__tests__/strict-type-guards.test.ts @@ -0,0 +1,106 @@ +/** + * Tests for the type assertions replaced with runtime validation (#228). + * + * Covers the three sites named in the issue: + * - config.ts STELLAR_NETWORK (was `as "testnet" | "mainnet"`) + * - lib/registry.ts sim.result (was `sim.result!`) + * - routes/admin.ts project_ids (was `raw as number[]`) + */ + +import * as fs from "fs"; +import * as path from "path"; + +import { isStellarNetwork, STELLAR_NETWORKS } from "../config"; + +const SRC_DIR = path.join(__dirname, ".."); + +function read(relativePath: string): string { + return fs.readFileSync(path.join(SRC_DIR, relativePath), "utf8"); +} + +describe("STELLAR_NETWORK validation (#228)", () => { + it("accepts only the two supported networks", () => { + expect(isStellarNetwork("testnet")).toBe(true); + expect(isStellarNetwork("mainnet")).toBe(true); + }); + + it("rejects anything else", () => { + for (const value of ["", "TESTNET", "futurenet", "main", "testnet ", "public"]) { + expect(isStellarNetwork(value)).toBe(false); + } + }); + + it("exposes the allowed values used for the startup error message", () => { + expect([...STELLAR_NETWORKS]).toEqual(["testnet", "mainnet"]); + }); + + it("coerces an unrecognised value to the default instead of trusting it", () => { + const originalEnv = process.env.STELLAR_NETWORK; + process.env.STELLAR_NETWORK = "not-a-network"; + jest.resetModules(); + try { + const fresh = jest.requireActual("../config"); + expect(fresh.config.STELLAR_NETWORK).toBe("testnet"); + expect(isStellarNetwork(fresh.config.STELLAR_NETWORK)).toBe(true); + } finally { + if (originalEnv === undefined) delete process.env.STELLAR_NETWORK; + else process.env.STELLAR_NETWORK = originalEnv; + jest.resetModules(); + } + }); + + it("validateRequiredEnv rejects an invalid network value", () => { + const original = { ...process.env }; + process.env.ADMIN_SECRET_KEY = "test-secret-key"; + process.env.PROJECT_REGISTRY_CONTRACT_ID = "C1234567890"; + process.env.STELLAR_NETWORK = "futurenet"; + try { + const { validateRequiredEnv } = jest.requireActual("../config"); + expect(() => validateRequiredEnv()).toThrow(/STELLAR_NETWORK/); + expect(() => validateRequiredEnv()).toThrow(/testnet, mainnet/); + } finally { + process.env = original; + } + }); + + it("validateRequiredEnv accepts both valid network values", () => { + const original = { ...process.env }; + process.env.ADMIN_SECRET_KEY = "test-secret-key"; + process.env.PROJECT_REGISTRY_CONTRACT_ID = "C1234567890"; + try { + const { validateRequiredEnv } = jest.requireActual("../config"); + for (const network of ["testnet", "mainnet"]) { + process.env.STELLAR_NETWORK = network; + expect(() => validateRequiredEnv()).not.toThrow(); + } + } finally { + process.env = original; + } + }); +}); + +describe("assertions removed at the sites named in #228", () => { + it("config.ts no longer casts STELLAR_NETWORK", () => { + expect(read("config.ts")).not.toMatch(/as\s+"testnet"\s*\|\s*"mainnet"/); + }); + + it("registry.ts no longer uses a non-null assertion on the simulation result", () => { + const content = read("lib/registry.ts"); + expect(content).not.toMatch(/sim\.result!/); + expect(content).not.toMatch(/\bas rpc\.Api\./); + // Replaced by an explicit undefined check. + expect(content).toMatch(/retval === undefined/); + }); + + it("admin.ts no longer casts the parsed project_ids array", () => { + const content = read("routes/admin.ts"); + expect(content).not.toMatch(/raw as number\[\]/); + expect(content).not.toMatch(/reason!/); + }); + + it("the three named files carry no non-null assertions at all", () => { + for (const file of ["config.ts", "lib/registry.ts", "lib/stellar.ts", "routes/admin.ts"]) { + expect(read(file)).not.toMatch(/\w!\./); + } + }); +}); diff --git a/src/config.ts b/src/config.ts index 6a6edf9..6ea6590 100644 --- a/src/config.ts +++ b/src/config.ts @@ -45,9 +45,34 @@ function validateEnvValue(name: string, value: string, allowedValues?: readonly } } +/** The Stellar networks this service knows how to talk to. */ +export type StellarNetwork = "testnet" | "mainnet"; + +export const STELLAR_NETWORKS: readonly StellarNetwork[] = ["testnet", "mainnet"]; + +/** + * Narrowing guard for STELLAR_NETWORK. Written as explicit comparisons so the + * check is a real runtime validation and TypeScript can derive the narrowed + * type without an assertion. + */ +export function isStellarNetwork(value: string): value is StellarNetwork { + return value === "testnet" || value === "mainnet"; +} + +/** + * Read a network-valued env var. An unset or unrecognised value falls back to + * `fallback` so importing this module never throws. `validateRequiredEnv` is + * what turns a misconfigured value into a startup error. + */ +function networkEnv(name: string, fallback: StellarNetwork): StellarNetwork { + const raw = process.env[name]; + if (!raw) return fallback; + return isStellarNetwork(raw) ? raw : fallback; +} + export const config = { /** Stellar / Soroban */ - STELLAR_NETWORK: optionalEnv("STELLAR_NETWORK", "testnet") as "testnet" | "mainnet", + STELLAR_NETWORK: networkEnv("STELLAR_NETWORK", "testnet"), ADMIN_SECRET_KEY: process.env.ADMIN_SECRET_KEY || "", PROJECT_REGISTRY_CONTRACT_ID: process.env.PROJECT_REGISTRY_CONTRACT_ID || "", RPC_URL: optionalEnv("RPC_URL", "https://soroban-testnet.stellar.org"), @@ -107,6 +132,12 @@ export const config = { SECRETS_PROVIDER: optionalEnv("SECRETS_PROVIDER", "env"), } as const; +/** + * The shape of the resolved application configuration. Exported so consumers + * and tests can reference the config type without re-deriving it inline. + */ +export type AppConfig = typeof config; + /** * Validate required environment variables at startup. * Call this once during server bootstrap before any routes or cron jobs. @@ -115,7 +146,10 @@ export const config = { export function validateRequiredEnv(): void { requireEnv("ADMIN_SECRET_KEY"); requireEnv("PROJECT_REGISTRY_CONTRACT_ID"); - validateEnvValue("STELLAR_NETWORK", config.STELLAR_NETWORK, ["testnet", "mainnet"]); + // Read the raw value rather than config.STELLAR_NETWORK: the config object is + // built once at import time and coerces unknown values to the default, so + // validating it would never see a bad value. + validateEnvValue("STELLAR_NETWORK", process.env.STELLAR_NETWORK || "", STELLAR_NETWORKS); } /** diff --git a/src/lib/registry.ts b/src/lib/registry.ts index 5ef3f57..54cb563 100644 --- a/src/lib/registry.ts +++ b/src/lib/registry.ts @@ -42,6 +42,17 @@ export async function updateImpactScore( }); } +/** + * Narrowing guard for a failed simulation. Defined locally rather than using + * the SDK's `rpc.Api.isSimulationError` so the check stays a plain shape test + * and does not depend on that helper being present at runtime. + */ +function isSimulationError( + sim: rpc.Api.SimulateTransactionResponse, +): sim is rpc.Api.SimulateTransactionErrorResponse { + return "error" in sim && typeof sim.error === "string"; +} + export async function getTotalProjects(): Promise { return withRpcConnection(async (client) => { const contract = new Contract(REGISTRY_CONTRACT_ID); @@ -55,10 +66,14 @@ export async function getTotalProjects(): Promise { .setTimeout(30) .build(); - const result = await client.simulateTransaction(tx); - if ("error" in result) throw new Error((result as { error: string }).error); - const sim = result as rpc.Api.SimulateTransactionSuccessResponse; - return Number(scValToNative(sim.result!.retval)); + const sim = await client.simulateTransaction(tx); + if (isSimulationError(sim)) throw new Error(sim.error); + + const retval = sim.result?.retval; + if (retval === undefined) { + throw new Error("total_projects simulation returned no result value"); + } + return Number(scValToNative(retval)); }); } diff --git a/src/lib/stellar.ts b/src/lib/stellar.ts index 9940652..191feaa 100644 --- a/src/lib/stellar.ts +++ b/src/lib/stellar.ts @@ -4,7 +4,8 @@ import { RpcConnectionPool } from "./db-pool"; import { CircuitBreaker } from "./circuit-breaker"; import { withRetry, isTransientError } from "./retry"; -export const networkPassphrase = config.STELLAR_NETWORK === "mainnet" ? Networks.PUBLIC : Networks.TESTNET; +export const networkPassphrase = + config.STELLAR_NETWORK === "mainnet" ? Networks.PUBLIC : Networks.TESTNET; export const rpcPool = new RpcConnectionPool({ rpcUrl: config.RPC_URL, @@ -67,10 +68,29 @@ export function withRpcConnection(fn: (client: rpc.Server) => Promise): Pr ); } +// ── Admin keypair cache (#227) ─────────────────────────────────────────────── +// Deriving an Ed25519 keypair from the secret is pure CPU work and the secret +// does not change during the process lifetime, so derive once and reuse. The +// secret it was derived from is cached alongside it: if the configured secret +// ever differs, the cache is rebuilt rather than handing back a stale keypair. +let cachedAdminKeypair: Keypair | null = null; +let cachedAdminSecret: string | null = null; + export function getAdminKeypair(): Keypair { const secretKey = config.ADMIN_SECRET_KEY; if (!secretKey) throw new Error("ADMIN_SECRET_KEY not set"); - return Keypair.fromSecret(secretKey); + + if (cachedAdminKeypair === null || cachedAdminSecret !== secretKey) { + cachedAdminKeypair = Keypair.fromSecret(secretKey); + cachedAdminSecret = secretKey; + } + return cachedAdminKeypair; +} + +/** Drop the cached keypair. Intended for tests that swap the configured secret. */ +export function resetAdminKeypairCache(): void { + cachedAdminKeypair = null; + cachedAdminSecret = null; } // ── FIXED SEQUENCE & CONCURRENCY QUEUE MANAGEMENT ─────────────────────────── @@ -113,16 +133,13 @@ async function _executeSignAndSubmitWithRetry( preparedXdr: string, keypair: Keypair, ): Promise { - return withRetry( - () => _attemptSubmit(client, preparedXdr, keypair), - { - maxAttempts: config.TX_MAX_RETRIES, - baseDelayMs: config.TX_RETRY_BASE_DELAY_MS, - maxDelayMs: config.TX_RETRY_MAX_DELAY_MS, - jitter: 0.3, - label: "stellar:signAndSubmit", - }, - ); + return withRetry(() => _attemptSubmit(client, preparedXdr, keypair), { + maxAttempts: config.TX_MAX_RETRIES, + baseDelayMs: config.TX_RETRY_BASE_DELAY_MS, + maxDelayMs: config.TX_RETRY_MAX_DELAY_MS, + jitter: 0.3, + label: "stellar:signAndSubmit", + }); } async function _attemptSubmit( @@ -190,7 +207,7 @@ async function _attemptSubmit( } // 2. Poll for confirmation - let getResult: rpc.Api.GetTransactionResponse; + let getResult: rpc.Api.GetTransactionResponse | undefined; let pollAttempts = 0; let timer: ReturnType | undefined; const pollIntervalMs = parseInt( @@ -211,7 +228,13 @@ async function _attemptSubmit( if (timer) clearTimeout(timer); } - if (getResult!.status === rpc.Api.GetTransactionStatus.FAILED) { + // The loop above always assigns before exiting normally, but the check keeps + // that a runtime guarantee rather than an assertion the compiler has to trust. + if (getResult === undefined) { + throw new Error("Transaction confirmation never returned a result"); + } + + if (getResult.status === rpc.Api.GetTransactionStatus.FAILED) { throw new Error("Transaction failed on-chain"); } diff --git a/src/routes/admin.ts b/src/routes/admin.ts index ca092b4..5e9c3fa 100644 --- a/src/routes/admin.ts +++ b/src/routes/admin.ts @@ -25,22 +25,70 @@ router.use((req: Request, res: Response, next: NextFunction) => { next(); }); +/** A per-project score update that made it onto the ledger (or was deferred). */ +type ScoreUpdateResult = { + project_id: number; + tx_hash: string; + credit_quality: number; + green_impact: number; +}; + +/** + * Discriminated result of one locked project update. The `skipped` flag lets + * the caller narrow the two shapes apart without a type assertion. + */ +type ProjectUpdateOutcome = + { skipped: true; reason: string } | ({ skipped: false } & ScoreUpdateResult); + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function isPositiveInteger(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value) && value >= 1; +} + +function isStringArray(value: unknown): value is string[] { + return Array.isArray(value) && value.every((entry) => typeof entry === "string"); +} + +/** + * Narrow an Express query value to what `parseOptionalInt` accepts. Express + * types query entries as a union that also covers nested objects; those are + * treated as absent rather than being asserted into a string. + */ +function queryValue(value: unknown): string | string[] | undefined { + if (typeof value === "string") return value; + if (isStringArray(value)) return value; + return undefined; +} + /** * Validate the optional `project_ids` field. Returns a list of ids, or `null` * to signal "update every registered project". Throws `ApiError` (400) on * anything that isn't an array of positive integers. + * + * Each entry is checked individually and copied into a `number[]`, so the + * returned array is typed by construction rather than by assertion. */ function parseProjectIds(body: unknown): number[] | null { - const raw = (body as { project_ids?: unknown } | undefined)?.project_ids; + if (!isRecord(body)) return null; + + const raw = body.project_ids; if (raw === undefined || raw === null) return null; if (!Array.isArray(raw)) { throw badRequest("project_ids must be an array of positive integers"); } if (raw.length === 0) return null; - if (!raw.every((n) => Number.isInteger(n) && (n as number) >= 1)) { - throw badRequest("project_ids must contain only positive integers"); + + const projectIds: number[] = []; + for (const entry of raw) { + if (!isPositiveInteger(entry)) { + throw badRequest("project_ids must contain only positive integers"); + } + projectIds.push(entry); } - return raw as number[]; + return projectIds; } // POST /api/admin/update-scores @@ -64,12 +112,7 @@ router.post("/update-scores", async (req: Request, res: Response, next: NextFunc projectIds = Array.from({ length: total }, (_, i) => i + 1); } - const results: Array<{ - project_id: number; - tx_hash: string; - credit_quality: number; - green_impact: number; - }> = []; + const results: ScoreUpdateResult[] = []; const errors: Array<{ project_id: number; error: { code: string; message: string } }> = []; const skipped: Array<{ project_id: number; reason: string }> = []; @@ -79,10 +122,10 @@ router.post("/update-scores", async (req: Request, res: Response, next: NextFunc // can retry only the affected ids. for (const projectId of projectIds) { try { - const result = await withProjectLock(projectId, async () => { + const result = await withProjectLock(projectId, async () => { const { allowed, reason } = tryBeginUpdate(projectId); if (!allowed) { - return { skipped: true, reason: reason! }; + return { skipped: true, reason }; } try { const scoreResult = await updateScoreForProject(projectId); @@ -91,6 +134,7 @@ router.post("/update-scores", async (req: Request, res: Response, next: NextFunc logger.warn(`[oracle] project ${projectId}: RPC degraded, score queued for later`); markCompleted(projectId); return { + skipped: false, project_id: projectId, tx_hash: "deferred", credit_quality: scoreResult.creditQuality, @@ -120,6 +164,7 @@ router.post("/update-scores", async (req: Request, res: Response, next: NextFunc `[oracle] project ${projectId}: cq=${scoreResult.creditQuality} gi=${scoreResult.greenImpact} tx=${scoreResult.txHash}`, ); return { + skipped: false, project_id: projectId, tx_hash: scoreResult.txHash, credit_quality: scoreResult.creditQuality, @@ -135,13 +180,14 @@ router.post("/update-scores", async (req: Request, res: Response, next: NextFunc skipped.push({ project_id: projectId, reason: result.reason }); logger.info(`[oracle] skipping project ${projectId}: ${result.reason}`); } else { - const r = result as { - project_id: number; - tx_hash: string; - credit_quality: number; - green_impact: number; - }; - results.push(r); + // Rebuilt field by field so the internal `skipped` discriminant does + // not leak into the response body. + results.push({ + project_id: result.project_id, + tx_hash: result.tx_hash, + credit_quality: result.credit_quality, + green_impact: result.green_impact, + }); } } catch (err) { logger.error(`[oracle] project ${projectId} failed`, logger.formatError(err)); @@ -149,7 +195,7 @@ router.post("/update-scores", async (req: Request, res: Response, next: NextFunc project_id: projectId, error: { code: "update_failed", - message: (err as Error)?.message || String(err), + message: err instanceof Error ? err.message : String(err), }, }); } @@ -171,9 +217,9 @@ router.post("/update-scores", async (req: Request, res: Response, next: NextFunc router.get("/audit", (req: Request, res: Response, next: NextFunction) => { try { const project_id = - parseOptionalInt(req.query.project_id as string | undefined, "project_id", 0) || undefined; - const from = parseOptionalInt(req.query.from as string | undefined, "from", 0) || undefined; - const to = parseOptionalInt(req.query.to as string | undefined, "to", 0) || undefined; + parseOptionalInt(queryValue(req.query.project_id), "project_id", 0) || undefined; + const from = parseOptionalInt(queryValue(req.query.from), "from", 0) || undefined; + const to = parseOptionalInt(queryValue(req.query.to), "to", 0) || undefined; if (from && to && from > to) { throw badRequest("from must be earlier than to"); @@ -195,4 +241,4 @@ router.get("/audit", (req: Request, res: Response, next: NextFunction) => { } }); -export default router; \ No newline at end of file +export default router;