feat: add request correlation, health checks, and graceful shutdown

.env.example

@@ -5,6 +5,13 @@ JWT_SECRET=your_jwt_secret
 # Database Configuration
 DATABASE_URL="postgresql://username:password@localhost:5432/learnault_db?schema=public"
 
+# Logging Configuration
+# LOG_LEVEL=info (options: error, warn, info, http, verbose, debug, silly)
+
+# Graceful Shutdown Configuration
+# Maximum time to wait for graceful shutdown before forcing exit (in milliseconds)
+SHUTDOWN_TIMEOUT_MS=30000
+
 # Stellar Funding Configuration
 STELLAR_FUNDING_AMOUNT=10
 STELLAR_FUNDING_MIN_BALANCE=1

integrations/unit/graceful-shutdown.test.ts

@@ -0,0 +1,146 @@
+import { describe, it, expect, vi, afterEach } from 'vitest'
+import { ChildProcess } from 'child_process'
+
+describe('Graceful Shutdown', () => {
+  const serverProcess: ChildProcess | null = null
+
+  afterEach(async () => {
+    if (serverProcess && !serverProcess.killed) {
+      serverProcess.kill('SIGTERM')
+      // Wait a bit for the process to shut down
+      await new Promise(resolve => setTimeout(resolve, 1000))
+    }
+  })
+
+  it('should handle SIGTERM and shutdown gracefully', async () => {
+    // This is a simulation test - in real scenario, you'd start the server
+    // and send SIGTERM to test graceful shutdown
+
+    const mockServer = {
+      close: vi.fn((callback) => callback()),
+    }
+
+    const mockPrisma = {
+      $disconnect: vi.fn().mockResolvedValue(undefined),
+    }
+
+    // Simulate graceful shutdown logic
+    await mockServer.close()
+    await mockPrisma.$disconnect()
+
+    expect(mockServer.close).toHaveBeenCalled()
+    expect(mockPrisma.$disconnect).toHaveBeenCalled()
+  }, 10000)
+
+  it('should handle SIGINT and shutdown gracefully', async () => {
+    const mockServer = {
+      close: vi.fn((callback) => callback()),
+    }
+
+    const mockPrisma = {
+      $disconnect: vi.fn().mockResolvedValue(undefined),
+    }
+
+    // Simulate graceful shutdown logic
+    await mockServer.close()
+    await mockPrisma.$disconnect()
+
+    expect(mockServer.close).toHaveBeenCalled()
+    expect(mockPrisma.$disconnect).toHaveBeenCalled()
+  }, 10000)
+
+  it('should stop background jobs during shutdown', async () => {
+    const mockInterval = {
+      clear: vi.fn(),
+    }
+
+    const lifecycleSweepInterval = setInterval(() => {}, 1000)
+
+    // Simulate clearing interval during shutdown
+    clearInterval(lifecycleSweepInterval)
+    mockInterval.clear()
+
+    expect(mockInterval.clear).toHaveBeenCalled()
+  })
+
+  it('should enforce shutdown timeout', async () => {
+    const SHUTDOWN_TIMEOUT_MS = 100
+
+    const mockServer = {
+      close: vi.fn((_callback) => {
+        // Simulate a server that never closes
+        // Don't call the callback
+      }),
+    }
+
+    const shutdownPromise = new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        reject(new Error('Shutdown timeout exceeded'))
+      }, SHUTDOWN_TIMEOUT_MS)
+
+      mockServer.close(() => {
+        clearTimeout(timer)
+        resolve(undefined)
+      })
+    })
+
+    await expect(shutdownPromise).rejects.toThrow('Shutdown timeout exceeded')
+  })
+
+  it('should handle shutdown errors gracefully', async () => {
+    const mockServer = {
+      close: vi.fn((callback) => callback(new Error('Server close error'))),
+    }
+
+    const mockPrisma = {
+      $disconnect: vi.fn().mockRejectedValue(new Error('Disconnect error')),
+    }
+
+    // Simulate graceful shutdown with errors
+    try {
+      await new Promise<void>((resolve, reject) => {
+        mockServer.close((err: Error | null) => {
+          if (err) reject(err)
+          else resolve()
+        })
+      })
+    } catch (error) {
+      expect(error).toBeDefined()
+      expect((error as Error).message).toBe('Server close error')
+    }
+
+    try {
+      await mockPrisma.$disconnect()
+    } catch (error) {
+      expect(error).toBeDefined()
+      expect((error as Error).message).toBe('Disconnect error')
+    }
+
+    expect(mockServer.close).toHaveBeenCalled()
+    expect(mockPrisma.$disconnect).toHaveBeenCalled()
+  })
+
+  it('should not process shutdown signal twice', async () => {
+    let isShuttingDown = false
+    let shutdownCount = 0
+
+    const gracefulShutdown = async (_signal: string) => {
+      if (isShuttingDown) {
+        return
+      }
+      isShuttingDown = true
+      shutdownCount++
+      // Simulate shutdown
+      await new Promise(resolve => setTimeout(resolve, 10))
+    }
+
+    // Simulate multiple signals
+    await Promise.all([
+      gracefulShutdown('SIGTERM'),
+      gracefulShutdown('SIGTERM'),
+      gracefulShutdown('SIGTERM'),
+    ])
+
+    expect(shutdownCount).toBe(1)
+  })
+})

integrations/unit/health.routes.test.ts

@@ -0,0 +1,121 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest'
+import request from 'supertest'
+import express from 'express'
+import healthRoutes from '../../src/routes/health.routes'
+import prisma from '../../src/config/database'
+
+// Mock the prisma module
+vi.mock('../../src/config/database', () => ({
+  default: {
+    $queryRaw: vi.fn(),
+  },
+}))
+
+describe('Health Routes', () => {
+  let app: express.Application
+
+  beforeEach(() => {
+    app = express()
+    app.use('/health', healthRoutes)
+    vi.clearAllMocks()
+  })
+
+  describe('GET /health/live', () => {
+    it('should return 200 with status ok', async () => {
+      const response = await request(app).get('/health/live')
+
+      expect(response.status).toBe(200)
+      expect(response.body.status).toBe('ok')
+      expect(response.body.timestamp).toBeDefined()
+    })
+
+    it('should return valid ISO timestamp', async () => {
+      const response = await request(app).get('/health/live')
+
+      const timestamp = new Date(response.body.timestamp)
+      expect(timestamp.toISOString()).toBe(response.body.timestamp)
+    })
+
+    it('should respond quickly', async () => {
+      const start = Date.now()
+      await request(app).get('/health/live')
+      const duration = Date.now() - start
+
+      // Should respond in less than 100ms (liveness checks should be fast)
+      expect(duration).toBeLessThan(100)
+    })
+  })
+
+  describe('GET /health/ready', () => {
+    it('should return 200 when all dependencies are healthy', async () => {
+      // Mock successful database query
+      vi.mocked(prisma.$queryRaw).mockResolvedValue([{ '?column?': 1 }])
+
+      const response = await request(app).get('/health/ready')
+
+      expect(response.status).toBe(200)
+      expect(response.body.status).toBe('ready')
+      expect(response.body.timestamp).toBeDefined()
+      expect(response.body.checks).toBeDefined()
+      expect(response.body.checks.database).toBe('ok')
+      expect(response.body.errors).toBeUndefined()
+    })
+
+    it('should return 503 when database is unavailable', async () => {
+      // Mock database failure
+      vi.mocked(prisma.$queryRaw).mockRejectedValue(new Error('Connection refused'))
+
+      const response = await request(app).get('/health/ready')
+
+      expect(response.status).toBe(503)
+      expect(response.body.status).toBe('not ready')
+      expect(response.body.checks.database).toBe('error')
+      expect(response.body.errors).toBeDefined()
+      expect(response.body.errors.length).toBeGreaterThan(0)
+      expect(response.body.errors[0]).toContain('Database connection failed')
+    })
+
+    it('should include error details in response', async () => {
+      const errorMessage = 'Timeout connecting to database'
+      vi.mocked(prisma.$queryRaw).mockRejectedValue(new Error(errorMessage))
+
+      const response = await request(app).get('/health/ready')
+
+      expect(response.status).toBe(503)
+      expect(response.body.errors).toBeDefined()
+      expect(response.body.errors[0]).toContain(errorMessage)
+    })
+
+    it('should return valid ISO timestamp', async () => {
+      vi.mocked(prisma.$queryRaw).mockResolvedValue([{ '?column?': 1 }])
+
+      const response = await request(app).get('/health/ready')
+
+      const timestamp = new Date(response.body.timestamp)
+      expect(timestamp.toISOString()).toBe(response.body.timestamp)
+    })
+
+    it('should include checks object in response', async () => {
+      vi.mocked(prisma.$queryRaw).mockResolvedValue([{ '?column?': 1 }])
+
+      const response = await request(app).get('/health/ready')
+
+      expect(response.body.checks).toBeDefined()
+      expect(typeof response.body.checks).toBe('object')
+      expect(response.body.checks.database).toBeDefined()
+    })
+  })
+
+  describe('Error Handling', () => {
+    it('should handle non-Error exceptions in database check', async () => {
+      // Mock rejection with non-Error object
+      vi.mocked(prisma.$queryRaw).mockRejectedValue('String error')
+
+      const response = await request(app).get('/health/ready')
+
+      expect(response.status).toBe(503)
+      expect(response.body.checks.database).toBe('error')
+      expect(response.body.errors).toBeDefined()
+    })
+  })
+})