Feature/production ready

README.md

@@ -66,6 +66,8 @@ curl http://localhost:3000/health/ready
 curl http://localhost:3000/metrics/json
 ```
 
+The container image health check now targets `GET /health/live`, so the runtime reports itself healthy to the scheduler when the process is up and the HTTP server is responding. Keep `GET /health/ready` for deeper dependency diagnosis and for platforms that support a separate readiness gate.
+
 ### Example authenticated checks
 
 ```bash

docs/architecture/current-architecture.md

@@ -93,6 +93,8 @@ Health is split into dedicated routes:
 - dependency drilldown → `GET /health/dependencies`
 - full deep view → `GET /health/deep`
 
+The container-level Docker `HEALTHCHECK` uses the liveness route so the scheduler can distinguish “process is running” from deeper dependency failures. Readiness and dependency drilldowns remain the stricter operational surfaces.
+
 Readiness returns non-200 when critical dependencies are unhealthy, including:
 
 - PostgreSQL connectivity

docs/runbooks/monitoring-alerts.md

@@ -186,7 +186,8 @@ Minimum dashboard panels:
 
 ## Operational notes
 
-- Treat `/health/ready` as the deployment/orchestration gate.
+- Treat the container `HEALTHCHECK` / `/health/live` signal as “the process is running and answering HTTP”.
+- Treat `/health/ready` as the stricter dependency/deployment gate when your platform supports a separate readiness concept.
 - Treat `/health/dependencies` and `/health/deep` as diagnosis surfaces, not just binary probes.
 - Prefer alerting on gauges that represent *current bad state* or *recent failure windows* instead of all-time counts.
 - When a Prometheus alert fires, capture the matching `operationId`, `correlationId`, or `nodeId` from logs and cluster-operation APIs for faster triage.

healthcheck.sh

@@ -1,17 +1,17 @@
 #!/bin/bash
 
 RESPONSE=$(curl -sS --connect-timeout 2 --max-time 4 -w "\n%{http_code}" \
-  http://127.0.0.1:3000/health/ready 2>/dev/null)
+  http://127.0.0.1:3000/health/live 2>/dev/null)
 
 HTTP_CODE=$(printf '%s\n' "$RESPONSE" | tail -n 1)
 BODY=$(printf '%s\n' "$RESPONSE" | sed '$d')
 
 if [ "$HTTP_CODE" = "200" ] && printf '%s' "$BODY" | grep -Eq '"status"[[:space:]]*:[[:space:]]*"ok"'; then
-    echo "[HEALTH] OK: readiness probe passed"
+    echo "[HEALTH] OK: liveness probe passed"
     exit 0
 fi
 
-echo "[HEALTH] FAIL: readiness probe failed (HTTP $HTTP_CODE)" >&2
+echo "[HEALTH] FAIL: liveness probe failed (HTTP $HTTP_CODE)" >&2
 if [ -n "$BODY" ]; then
     printf '%s\n' "$BODY" >&2
 fi

src/certificate/certificate.service.ts

@@ -55,6 +55,7 @@ class ArtifactActivationError extends Error {
 export class CertificateService implements OnModuleInit, OnApplicationShutdown {
   private readonly logger = new Logger(CertificateService.name);
   private interval: NodeJS.Timeout | null = null;
+  private syncInterval: NodeJS.Timeout | null = null;
   private readonly renewIntervalMs = 1000 * 60 * 60 * 12; // 12 hours
   private leaderLockInterval: NodeJS.Timeout | null = null;
   private isCurrentlyLeader = false;
@@ -109,7 +110,7 @@ export class CertificateService implements OnModuleInit, OnApplicationShutdown {
     );
 
     // Schedule periodic sync (every 5 minutes)
-    setInterval(
+    this.syncInterval = setInterval(
       () => {
         this.syncCertificates().catch((err) =>
           this.logger.error(
@@ -325,6 +326,11 @@ export class CertificateService implements OnModuleInit, OnApplicationShutdown {
       this.interval = null;
     }
 
+    if (this.syncInterval) {
+      clearInterval(this.syncInterval);
+      this.syncInterval = null;
+    }
+
     if (this.leaderLockInterval) {
       clearInterval(this.leaderLockInterval);
       this.leaderLockInterval = null;

src/health/health.service.ts

@@ -1,12 +1,15 @@
-import { Injectable, Logger } from '@nestjs/common';
+import { BeforeApplicationShutdown, Injectable, Logger } from '@nestjs/common';
 import * as fsPromises from 'node:fs/promises';
 import * as process from 'node:process';
 import { PrismaService } from '../prisma/prisma.service';
 
 @Injectable()
-export class HealthService {
+export class HealthService implements BeforeApplicationShutdown {
   private readonly logger = new Logger(HealthService.name);
   private readonly startedAt = new Date();
+  private lifecycleState: LifecycleState = 'starting';
+  private lifecycleChangedAt = new Date();
+  private shutdownSignal: string | null = null;
   private readonly configApplyMaxAgeMs = this.getThresholdFromEnv(
     'HEALTH_CONFIG_APPLY_MAX_AGE_MS',
     15 * 60 * 1000,
@@ -20,17 +23,35 @@ export class HealthService {
 
   constructor(private readonly prisma: PrismaService) {}
 
+  markRunning(details = 'http server is accepting connections') {
+    this.transitionLifecycle('running', details);
+  }
+
+  beforeApplicationShutdown(signal?: string) {
+    this.transitionLifecycle(
+      'shutting_down',
+      signal
+        ? `shutdown signal ${signal} received`
+        : 'application shutdown requested',
+      signal,
+    );
+  }
+
   async live() {
+    const lifecycle = this.createLifecycleSnapshot();
+
     return {
-      status: 'ok',
+      status: lifecycle.state === 'running' ? 'ok' : lifecycle.status,
       probe: 'liveness',
+      lifecycle,
       uptime: process.uptime(),
       startedAt: this.startedAt.toISOString(),
       timestamp: new Date().toISOString(),
     };
   }
 
   async startup() {
+    const lifecycle = this.createLifecycleSnapshot();
     const checks = [
       this.checkInitializationState('config_apply', this.configApplyState),
       this.checkInitializationState(
@@ -44,6 +65,7 @@ export class HealthService {
     return {
       status: healthy ? 'ok' : 'starting',
       probe: 'startup',
+      lifecycle,
       checks,
       uptime: process.uptime(),
       startedAt: this.startedAt.toISOString(),
@@ -352,10 +374,12 @@ export class HealthService {
   ) {
     const healthy = checks.every((check) => check.status === 'ok');
     const summary = this.summarizeChecks(checks);
+    const lifecycle = this.createLifecycleSnapshot();
 
     return {
       status: healthy ? 'ok' : 'error',
       probe,
+      lifecycle,
       checks,
       summary,
       thresholds: {
@@ -379,6 +403,41 @@ export class HealthService {
     };
   }
 
+  private transitionLifecycle(
+    nextState: LifecycleState,
+    message: string,
+    signal?: string,
+  ) {
+    if (this.lifecycleState === nextState) {
+      if (signal) {
+        this.shutdownSignal = signal;
+      }
+      return;
+    }
+
+    this.lifecycleState = nextState;
+    this.lifecycleChangedAt = new Date();
+    this.shutdownSignal = signal ?? null;
+    this.logger.log(`[Lifecycle] ${message}`);
+  }
+
+  private createLifecycleSnapshot() {
+    return {
+      state: this.lifecycleState,
+      status: this.lifecycleState === 'running' ? 'ok' : this.mapLifecycleStatus(),
+      changedAt: this.lifecycleChangedAt.toISOString(),
+      shutdownSignal: this.shutdownSignal,
+    };
+  }
+
+  private mapLifecycleStatus(): ProbeReportStatus {
+    if (this.lifecycleState === 'starting') {
+      return 'starting';
+    }
+
+    return 'stopping';
+  }
+
   private createOperationSnapshot(
     name: string,
     state: OperationState,
@@ -422,6 +481,10 @@ export class HealthService {
 
 type HealthCheckStatus = 'ok' | 'error';
 
+type ProbeReportStatus = 'ok' | 'error' | 'starting' | 'stopping';
+
+type LifecycleState = 'starting' | 'running' | 'shutting_down';
+
 type OperationState = {
   lastAttemptAt: Date | null;
   lastSuccessAt: Date | null;

src/main.ts

@@ -2,13 +2,16 @@ import { NestFactory } from '@nestjs/core';
 import { ValidationPipe } from '@nestjs/common';
 import { Response } from 'express';
 import { AppModule } from './app.module';
+import { HealthService } from './health/health.service';
 import { LogsService } from './logs/logs.service';
 import { GlobalExceptionFilter } from './filters/global-exception.filter';
 import { AuthenticatedRequest } from './auth/interfaces/authenticated-request.interface';
 
 async function bootstrap() {
   const app = await NestFactory.create(AppModule, { bufferLogs: true });
+  app.enableShutdownHooks();
   const logsService = app.get(LogsService);
+  const healthService = app.get(HealthService);
   app.useLogger(logsService);
   app.use(
     (
@@ -34,6 +37,7 @@ async function bootstrap() {
   );
 
   await app.listen(process.env['PORT'] ?? 3000);
+  healthService.markRunning();
 }
 
 void bootstrap();

test/unit/health-semantics.test.js

@@ -75,9 +75,16 @@ describe('health service semantics', () => {
     };
     const service = new HealthService(prisma);
 
+    const preListenLiveReport = await service.live();
+    assert.equal(preListenLiveReport.status, 'starting');
+    assert.equal(preListenLiveReport.lifecycle.state, 'starting');
+
+    service.markRunning();
+
     const liveReport = await service.live();
     assert.equal(liveReport.status, 'ok');
     assert.equal(liveReport.probe, 'liveness');
+    assert.equal(liveReport.lifecycle.state, 'running');
 
     const startupReport = await service.startup();
     assert.equal(startupReport.status, 'starting');
@@ -132,6 +139,23 @@ describe('health service semantics', () => {
     );
   });
 
+  it('marks liveness as stopping during graceful shutdown', async () => {
+    stubHealthyNginx(5757);
+
+    const prisma = {
+      $queryRawUnsafe: async () => [{ '?column?': 1 }],
+    };
+    const service = new HealthService(prisma);
+
+    service.markRunning();
+    service.beforeApplicationShutdown('SIGTERM');
+
+    const liveReport = await service.live();
+    assert.equal(liveReport.status, 'stopping');
+    assert.equal(liveReport.lifecycle.state, 'shutting_down');
+    assert.equal(liveReport.lifecycle.shutdownSignal, 'SIGTERM');
+  });
+
   it('fails readiness when the latest successful state is superseded by a failed attempt', async () => {
     stubHealthyNginx(6262);