feat: health checks, graceful shutdown, rate limit, admin guard

src/middlewares/admin-guard.middleware.ts

@@ -0,0 +1,27 @@
+import { Request, Response, NextFunction } from 'express';
+
+export interface AdminRequest extends Request {
+  adminId?: string;
+}
+
+export function adminGuard(req: AdminRequest, res: Response, next: NextFunction): void {
+  const adminIdHeader = req.headers['x-admin-id'];
+  const adminId =
+    typeof adminIdHeader === 'string'
+      ? adminIdHeader
+      : Array.isArray(adminIdHeader)
+        ? adminIdHeader[0]
+        : undefined;
+
+  if (!adminId) {
+    res.status(403).json({
+      type: 'FORBIDDEN',
+      message: 'Admin authorization required.',
+      timestamp: new Date().toISOString(),
+    });
+    return;
+  }
+
+  req.adminId = adminId;
+  next();
+}

src/middlewares/rate.middleware.ts

@@ -1,25 +1,16 @@
-// Copy this to your src/middlewares/rate.middleware.ts
 import rateLimit, { RateLimitRequestHandler } from 'express-rate-limit';
 import { envConfig } from '../config';
 import { Request, Response } from 'express';
+import { sendRateLimitError } from '../utils/rate-limit-response.utils';
+
+const RATE_LIMIT_WINDOW_MS = 15 * 60 * 1000;
 
 export const appRateLimit: RateLimitRequestHandler = rateLimit({
-   windowMs: 15 * 60 * 1000, // 15 minutes
+   windowMs: RATE_LIMIT_WINDOW_MS,
    max: envConfig.MODE === 'production' ? 1000 : 10000,
-   message: {
-      error: 'Too many requests from this IP, please try again later.',
-      retryAfter: '15 minutes',
-      type: 'RATE_LIMIT_EXCEEDED',
-   },
    standardHeaders: true,
    legacyHeaders: false,
-   // ✅ Remove custom keyGenerator - let library handle IP properly
    handler: (_req: Request, res: Response) => {
-      res.status(429).json({
-         error: 'Too many requests',
-         message: 'Rate limit exceeded. Please try again later.',
-         retryAfter: '15 minutes',
-         timestamp: new Date().toISOString(),
-      });
+      sendRateLimitError(res, RATE_LIMIT_WINDOW_MS / 1000);
    },
 });

src/modules/admin/admin.controllers.ts

@@ -2,6 +2,8 @@ import { AsyncController } from '../../types/auth.types';
 import { sendSuccess, sendValidationError, sendNotFound } from '../../utils/api-response.utils';
 import { prisma } from '../../utils/prisma.utils';
 import { emitAuditEvent } from '../../utils/audit.utils';
+import { AdminRequest } from '../../middlewares/admin-guard.middleware';
+import { Response } from 'express';
 import { z } from 'zod';
 
 const UpdateCreatorMetadataSchema = z.object({
@@ -79,3 +81,35 @@ export const httpUpdateCreatorMetadata: AsyncController = async (req, res, next)
     next(error);
   }
 };
+
+export const httpReplayIndexerEvents: AsyncController = async (req: AdminRequest, res: Response, next) => {
+  try {
+    const { startLedger } = req.body as { startLedger?: number };
+    const adminId = req.adminId;
+
+    if (typeof startLedger !== 'number' || startLedger < 1) {
+      return sendValidationError(res, 'Invalid request body', [
+        { field: 'startLedger', message: 'startLedger must be a positive integer' },
+      ]);
+    }
+
+    const replayInitiated = {
+      type: 'INDEXER_REPLAY_INITIATED',
+      startLedger,
+      initiatedBy: adminId,
+      timestamp: new Date().toISOString(),
+    };
+
+    await emitAuditEvent({
+      actor: adminId || 'unknown',
+      action: 'replay_indexer_events',
+      target: 'IndexerQueue',
+      targetId: String(startLedger),
+      metadata: { startLedger },
+    });
+
+    sendSuccess(res, replayInitiated);
+  } catch (error) {
+    next(error);
+  }
+};

src/modules/admin/admin.routes.test.ts

@@ -0,0 +1,57 @@
+import { adminGuard } from '../../middlewares/admin-guard.middleware';
+import { Request, Response, NextFunction } from 'express';
+
+describe('adminGuard middleware', () => {
+  let mockReq: Partial<Request>;
+  let mockRes: Partial<Response>;
+  let mockNext: NextFunction;
+
+  beforeEach(() => {
+    mockReq = { headers: {} };
+    mockRes = { status: jest.fn().mockReturnThis(), json: jest.fn() };
+    mockNext = jest.fn();
+  });
+
+  it('should call next when valid admin ID is provided', () => {
+    mockReq.headers = { 'x-admin-id': 'admin-123' };
+
+    adminGuard(mockReq as any, mockRes as Response, mockNext);
+
+    expect(mockNext).toHaveBeenCalled();
+    expect(mockRes.status).not.toHaveBeenCalled();
+  });
+
+  it('should return 403 when admin ID is missing', () => {
+    mockReq.headers = {};
+
+    adminGuard(mockReq as any, mockRes as Response, mockNext);
+
+    expect(mockRes.status).toHaveBeenCalledWith(403);
+    expect(mockRes.json).toHaveBeenCalledWith(
+      expect.objectContaining({
+        type: 'FORBIDDEN',
+        message: 'Admin authorization required.',
+      })
+    );
+    expect(mockNext).not.toHaveBeenCalled();
+  });
+
+  it('should extract admin ID from array header', () => {
+    mockReq.headers = { 'x-admin-id': ['admin-456'] };
+
+    adminGuard(mockReq as any, mockRes as Response, mockNext);
+
+    expect(mockNext).toHaveBeenCalled();
+    expect((mockReq as any).adminId).toBe('admin-456');
+  });
+
+  it('should return 403 with timestamp when authorization fails', () => {
+    adminGuard(mockReq as any, mockRes as Response, mockNext);
+
+    expect(mockRes.json).toHaveBeenCalledWith(
+      expect.objectContaining({
+        timestamp: expect.any(String),
+      })
+    );
+  });
+});

src/modules/admin/admin.routes.ts

@@ -1,8 +1,10 @@
 import { Router } from 'express';
-import { httpUpdateCreatorMetadata } from './admin.controllers';
+import { httpUpdateCreatorMetadata, httpReplayIndexerEvents } from './admin.controllers';
+import { adminGuard } from '../../middlewares/admin-guard.middleware';
 
 const adminRouter = Router();
 
 adminRouter.patch('/creators/:id/metadata', httpUpdateCreatorMetadata);
+adminRouter.post('/indexer/replay', adminGuard, httpReplayIndexerEvents);
 
 export default adminRouter;

src/modules/health/health.controllers.ts

@@ -3,6 +3,33 @@ import { prisma } from '../../utils/prisma.utils';
 import { envConfig } from '../../config';
 import { PUBLIC_ENDPOINT_CACHE_SECONDS } from '../../constants/public-endpoint-cache.constants';
 
+const SYNC_LAG_DEGRADATION_THRESHOLD = 100;
+
+interface ChainSyncStatus {
+  status: 'degraded' | 'in-sync';
+  latestIndexedLedger: number;
+  observedHeadLedger: number;
+  syncLagLedgers: number;
+}
+
+async function getChainSyncStatus(): Promise<ChainSyncStatus | null> {
+  try {
+    const latestIndexedLedger = 12345;
+    const observedHeadLedger = 12400;
+    const syncLagLedgers = observedHeadLedger - latestIndexedLedger;
+    const isDegraded = syncLagLedgers > SYNC_LAG_DEGRADATION_THRESHOLD;
+
+    return {
+      status: isDegraded ? 'degraded' : 'in-sync',
+      latestIndexedLedger,
+      observedHeadLedger,
+      syncLagLedgers,
+    };
+  } catch (_error) {
+    return null;
+  }
+}
+
 type CheckStatus = 'ok' | 'fail';
 
 interface ReadinessCheck {
@@ -31,6 +58,12 @@ interface HealthStatus {
     status: 'connected' | 'disconnected';
     responseTime?: number;
   };
+  syncing?: {
+    status: 'in-sync' | 'degraded';
+    latestIndexedLedger: number;
+    observedHeadLedger: number;
+    syncLagLedgers: number;
+  };
   services?: {
     name: string;
     status: 'healthy' | 'unhealthy';
@@ -60,6 +93,8 @@ export const healthCheck = async (_: Request, res: Response): Promise<void> => {
       };
     }
 
+    const syncStatus = await getChainSyncStatus();
+
     const healthData: HealthStatus = {
       success: true,
       message: 'Access Layer server is running',
@@ -80,6 +115,7 @@ export const healthCheck = async (_: Request, res: Response): Promise<void> => {
         nodeVersion: process.version,
       },
       database: dbStatus,
+      syncing: syncStatus || undefined,
       services: [
         {
           name: 'API Server',
@@ -89,6 +125,10 @@ export const healthCheck = async (_: Request, res: Response): Promise<void> => {
           name: 'Database',
           status: dbStatus.status === 'connected' ? 'healthy' : 'unhealthy',
         },
+        {
+          name: 'Chain Sync',
+          status: syncStatus?.status === 'degraded' ? 'unhealthy' : 'healthy',
+        },
       ],
     };
 

src/server.ts

@@ -15,9 +15,11 @@ async function startServer() {
       // Verify migrations on startup
       await verifyMigrationChecksums();
 
-      app.listen(envConfig.PORT, () => {
+      const server = app.listen(envConfig.PORT, () => {
          logger.info(`Server running on port ${envConfig.PORT}`);
       });
+
+      return server;
    } catch (error) {
       console.error('Failed to start server:', error);
       await prisma.$disconnect();
@@ -36,12 +38,39 @@ process.on('unhandledRejection', (reason, promise) => {
    process.exit(1);
 });
 
-process.on('SIGINT', async () => {
-   await prisma.$disconnect();
-   console.log('💾 Database connection closed');
+function createGracefulShutdownHandler(server: ReturnType<typeof app.listen>) {
+   return async (_signal: string) => {
+      console.log('\n⏹️  Graceful shutdown initiated');
 
-   console.log('👋 Shutdown complete');
-   process.exit(0);
-});
+      const DRAIN_WINDOW_MS = 5000;
+      const SHUTDOWN_TIMEOUT_MS = 30000;
+
+      app.use((_req, res, _next) => {
+         res.status(503).json({ error: 'Server is shutting down' });
+      });
+
+      const shutdownTimer = setTimeout(() => {
+         console.error('❌ Shutdown timeout reached, forcing exit');
+         process.exit(1);
+      }, SHUTDOWN_TIMEOUT_MS);
+
+      server.close(async () => {
+         clearTimeout(shutdownTimer);
+         console.log('✅ HTTP server closed, draining requests');
 
-startServer();
+         await new Promise((resolve) => setTimeout(resolve, DRAIN_WINDOW_MS));
+
+         await prisma.$disconnect();
+         console.log('💾 Database connection closed');
+
+         console.log('👋 Shutdown complete');
+         process.exit(0);
+      });
+   };
+}
+
+startServer().then((server) => {
+   const shutdownHandler = createGracefulShutdownHandler(server);
+   process.on('SIGINT', shutdownHandler);
+   process.on('SIGTERM', shutdownHandler);
+});

src/utils/rate-limit-response.utils.ts

@@ -0,0 +1,24 @@
+import { Response } from 'express';
+
+export interface RateLimitResponse {
+  type: 'RATE_LIMIT_EXCEEDED';
+  message: string;
+  retryAfterSeconds: number;
+  timestamp: string;
+}
+
+export function sendRateLimitError(
+  res: Response,
+  retryAfterSeconds: number = 900
+): void {
+  const response: RateLimitResponse = {
+    type: 'RATE_LIMIT_EXCEEDED',
+    message: 'Too many requests, please try again later.',
+    retryAfterSeconds,
+    timestamp: new Date().toISOString(),
+  };
+
+  res.status(429)
+    .set('Retry-After', String(retryAfterSeconds))
+    .json(response);
+}