diff --git a/README.md b/README.md index be29ff8..2307639 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,8 @@ curl http://localhost:3000/health/ready curl http://localhost:3000/metrics/json ``` +The container image health check now targets `GET /health/live`, so the runtime reports itself healthy to the scheduler when the process is up and the HTTP server is responding. Keep `GET /health/ready` for deeper dependency diagnosis and for platforms that support a separate readiness gate. + ### Example authenticated checks ```bash diff --git a/docs/architecture/current-architecture.md b/docs/architecture/current-architecture.md index 47cbee7..7e3f08f 100644 --- a/docs/architecture/current-architecture.md +++ b/docs/architecture/current-architecture.md @@ -93,6 +93,8 @@ Health is split into dedicated routes: - dependency drilldown → `GET /health/dependencies` - full deep view → `GET /health/deep` +The container-level Docker `HEALTHCHECK` uses the liveness route so the scheduler can distinguish “process is running” from deeper dependency failures. Readiness and dependency drilldowns remain the stricter operational surfaces. + Readiness returns non-200 when critical dependencies are unhealthy, including: - PostgreSQL connectivity diff --git a/docs/runbooks/monitoring-alerts.md b/docs/runbooks/monitoring-alerts.md index b850649..5cc2e22 100644 --- a/docs/runbooks/monitoring-alerts.md +++ b/docs/runbooks/monitoring-alerts.md @@ -186,7 +186,8 @@ Minimum dashboard panels: ## Operational notes -- Treat `/health/ready` as the deployment/orchestration gate. +- Treat the container `HEALTHCHECK` / `/health/live` signal as “the process is running and answering HTTP”. +- Treat `/health/ready` as the stricter dependency/deployment gate when your platform supports a separate readiness concept. - Treat `/health/dependencies` and `/health/deep` as diagnosis surfaces, not just binary probes. - Prefer alerting on gauges that represent *current bad state* or *recent failure windows* instead of all-time counts. - When a Prometheus alert fires, capture the matching `operationId`, `correlationId`, or `nodeId` from logs and cluster-operation APIs for faster triage. diff --git a/healthcheck.sh b/healthcheck.sh index 677ebf8..64513bf 100644 --- a/healthcheck.sh +++ b/healthcheck.sh @@ -1,17 +1,17 @@ #!/bin/bash RESPONSE=$(curl -sS --connect-timeout 2 --max-time 4 -w "\n%{http_code}" \ - http://127.0.0.1:3000/health/ready 2>/dev/null) + http://127.0.0.1:3000/health/live 2>/dev/null) HTTP_CODE=$(printf '%s\n' "$RESPONSE" | tail -n 1) BODY=$(printf '%s\n' "$RESPONSE" | sed '$d') if [ "$HTTP_CODE" = "200" ] && printf '%s' "$BODY" | grep -Eq '"status"[[:space:]]*:[[:space:]]*"ok"'; then - echo "[HEALTH] OK: readiness probe passed" + echo "[HEALTH] OK: liveness probe passed" exit 0 fi -echo "[HEALTH] FAIL: readiness probe failed (HTTP $HTTP_CODE)" >&2 +echo "[HEALTH] FAIL: liveness probe failed (HTTP $HTTP_CODE)" >&2 if [ -n "$BODY" ]; then printf '%s\n' "$BODY" >&2 fi diff --git a/src/certificate/certificate.service.ts b/src/certificate/certificate.service.ts index f7cae7a..39bba77 100644 --- a/src/certificate/certificate.service.ts +++ b/src/certificate/certificate.service.ts @@ -55,6 +55,7 @@ class ArtifactActivationError extends Error { export class CertificateService implements OnModuleInit, OnApplicationShutdown { private readonly logger = new Logger(CertificateService.name); private interval: NodeJS.Timeout | null = null; + private syncInterval: NodeJS.Timeout | null = null; private readonly renewIntervalMs = 1000 * 60 * 60 * 12; // 12 hours private leaderLockInterval: NodeJS.Timeout | null = null; private isCurrentlyLeader = false; @@ -109,7 +110,7 @@ export class CertificateService implements OnModuleInit, OnApplicationShutdown { ); // Schedule periodic sync (every 5 minutes) - setInterval( + this.syncInterval = setInterval( () => { this.syncCertificates().catch((err) => this.logger.error( @@ -325,6 +326,11 @@ export class CertificateService implements OnModuleInit, OnApplicationShutdown { this.interval = null; } + if (this.syncInterval) { + clearInterval(this.syncInterval); + this.syncInterval = null; + } + if (this.leaderLockInterval) { clearInterval(this.leaderLockInterval); this.leaderLockInterval = null; diff --git a/src/health/health.service.ts b/src/health/health.service.ts index 3d1fb5c..a892899 100644 --- a/src/health/health.service.ts +++ b/src/health/health.service.ts @@ -1,12 +1,15 @@ -import { Injectable, Logger } from '@nestjs/common'; +import { BeforeApplicationShutdown, Injectable, Logger } from '@nestjs/common'; import * as fsPromises from 'node:fs/promises'; import * as process from 'node:process'; import { PrismaService } from '../prisma/prisma.service'; @Injectable() -export class HealthService { +export class HealthService implements BeforeApplicationShutdown { private readonly logger = new Logger(HealthService.name); private readonly startedAt = new Date(); + private lifecycleState: LifecycleState = 'starting'; + private lifecycleChangedAt = new Date(); + private shutdownSignal: string | null = null; private readonly configApplyMaxAgeMs = this.getThresholdFromEnv( 'HEALTH_CONFIG_APPLY_MAX_AGE_MS', 15 * 60 * 1000, @@ -20,10 +23,27 @@ export class HealthService { constructor(private readonly prisma: PrismaService) {} + markRunning(details = 'http server is accepting connections') { + this.transitionLifecycle('running', details); + } + + beforeApplicationShutdown(signal?: string) { + this.transitionLifecycle( + 'shutting_down', + signal + ? `shutdown signal ${signal} received` + : 'application shutdown requested', + signal, + ); + } + async live() { + const lifecycle = this.createLifecycleSnapshot(); + return { - status: 'ok', + status: lifecycle.state === 'running' ? 'ok' : lifecycle.status, probe: 'liveness', + lifecycle, uptime: process.uptime(), startedAt: this.startedAt.toISOString(), timestamp: new Date().toISOString(), @@ -31,6 +51,7 @@ export class HealthService { } async startup() { + const lifecycle = this.createLifecycleSnapshot(); const checks = [ this.checkInitializationState('config_apply', this.configApplyState), this.checkInitializationState( @@ -44,6 +65,7 @@ export class HealthService { return { status: healthy ? 'ok' : 'starting', probe: 'startup', + lifecycle, checks, uptime: process.uptime(), startedAt: this.startedAt.toISOString(), @@ -352,10 +374,12 @@ export class HealthService { ) { const healthy = checks.every((check) => check.status === 'ok'); const summary = this.summarizeChecks(checks); + const lifecycle = this.createLifecycleSnapshot(); return { status: healthy ? 'ok' : 'error', probe, + lifecycle, checks, summary, thresholds: { @@ -379,6 +403,41 @@ export class HealthService { }; } + private transitionLifecycle( + nextState: LifecycleState, + message: string, + signal?: string, + ) { + if (this.lifecycleState === nextState) { + if (signal) { + this.shutdownSignal = signal; + } + return; + } + + this.lifecycleState = nextState; + this.lifecycleChangedAt = new Date(); + this.shutdownSignal = signal ?? null; + this.logger.log(`[Lifecycle] ${message}`); + } + + private createLifecycleSnapshot() { + return { + state: this.lifecycleState, + status: this.lifecycleState === 'running' ? 'ok' : this.mapLifecycleStatus(), + changedAt: this.lifecycleChangedAt.toISOString(), + shutdownSignal: this.shutdownSignal, + }; + } + + private mapLifecycleStatus(): ProbeReportStatus { + if (this.lifecycleState === 'starting') { + return 'starting'; + } + + return 'stopping'; + } + private createOperationSnapshot( name: string, state: OperationState, @@ -422,6 +481,10 @@ export class HealthService { type HealthCheckStatus = 'ok' | 'error'; +type ProbeReportStatus = 'ok' | 'error' | 'starting' | 'stopping'; + +type LifecycleState = 'starting' | 'running' | 'shutting_down'; + type OperationState = { lastAttemptAt: Date | null; lastSuccessAt: Date | null; diff --git a/src/main.ts b/src/main.ts index a6413a9..de310ed 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2,13 +2,16 @@ import { NestFactory } from '@nestjs/core'; import { ValidationPipe } from '@nestjs/common'; import { Response } from 'express'; import { AppModule } from './app.module'; +import { HealthService } from './health/health.service'; import { LogsService } from './logs/logs.service'; import { GlobalExceptionFilter } from './filters/global-exception.filter'; import { AuthenticatedRequest } from './auth/interfaces/authenticated-request.interface'; async function bootstrap() { const app = await NestFactory.create(AppModule, { bufferLogs: true }); + app.enableShutdownHooks(); const logsService = app.get(LogsService); + const healthService = app.get(HealthService); app.useLogger(logsService); app.use( ( @@ -34,6 +37,7 @@ async function bootstrap() { ); await app.listen(process.env['PORT'] ?? 3000); + healthService.markRunning(); } void bootstrap(); diff --git a/test/unit/health-semantics.test.js b/test/unit/health-semantics.test.js index 82deafc..25ef8c6 100644 --- a/test/unit/health-semantics.test.js +++ b/test/unit/health-semantics.test.js @@ -75,9 +75,16 @@ describe('health service semantics', () => { }; const service = new HealthService(prisma); + const preListenLiveReport = await service.live(); + assert.equal(preListenLiveReport.status, 'starting'); + assert.equal(preListenLiveReport.lifecycle.state, 'starting'); + + service.markRunning(); + const liveReport = await service.live(); assert.equal(liveReport.status, 'ok'); assert.equal(liveReport.probe, 'liveness'); + assert.equal(liveReport.lifecycle.state, 'running'); const startupReport = await service.startup(); assert.equal(startupReport.status, 'starting'); @@ -132,6 +139,23 @@ describe('health service semantics', () => { ); }); + it('marks liveness as stopping during graceful shutdown', async () => { + stubHealthyNginx(5757); + + const prisma = { + $queryRawUnsafe: async () => [{ '?column?': 1 }], + }; + const service = new HealthService(prisma); + + service.markRunning(); + service.beforeApplicationShutdown('SIGTERM'); + + const liveReport = await service.live(); + assert.equal(liveReport.status, 'stopping'); + assert.equal(liveReport.lifecycle.state, 'shutting_down'); + assert.equal(liveReport.lifecycle.shutdownSignal, 'SIGTERM'); + }); + it('fails readiness when the latest successful state is superseded by a failed attempt', async () => { stubHealthyNginx(6262);