Security and resilience improvements

services/api-gateway/src/health-all.test.ts

@@ -59,6 +59,41 @@ test('GET /api/health/all aggregates downstream health with graceful degradation
   await app.close();
 });
 
+test('HEAD /api/health includes security hardening headers', async () => {
+  const prisma = createMockPrisma() as any;
+
+  const app = buildApp({
+    prisma,
+    logger: false,
+    fetchImpl: (async () => jsonResponse(mockHealth('service', 'healthy'))) as any
+  });
+
+  const res = await app.inject({ method: 'HEAD', url: '/api/health' });
+
+  assert.equal(res.statusCode, 200);
+
+  assert.ok(res.headers['strict-transport-security']);
+  assert.ok(res.headers['x-content-type-options']);
+  assert.ok(res.headers['x-download-options']);
+  assert.ok(res.headers['x-frame-options']);
+  assert.ok(res.headers['x-permitted-cross-domain-policies']);
+  assert.ok(res.headers['x-xss-protection']);
+  assert.ok(res.headers['cross-origin-embedder-policy']);
+  assert.ok(res.headers['cross-origin-opener-policy']);
+  assert.ok(res.headers['cross-origin-resource-policy']);
+  assert.ok(res.headers['referrer-policy']);
+  assert.ok(res.headers['permissions-policy']);
+
+  assert.equal(res.headers['cross-origin-embedder-policy'], 'require-corp');
+  assert.equal(res.headers['cross-origin-opener-policy'], 'same-origin');
+  assert.equal(res.headers['cross-origin-resource-policy'], 'same-origin');
+  assert.equal(res.headers['referrer-policy'], 'strict-origin-when-cross-origin');
+  assert.equal(res.headers['permissions-policy'], 'geolocation=(), microphone=(), camera=()');
+  assert.ok((res.headers['strict-transport-security'] as string).includes('max-age=31536000'));
+
+  await app.close();
+});
+
 test('GET /api/health returns gateway dependency and upstream probes', async () => {
   const prisma = createMockPrisma() as any;
 

services/api-gateway/src/index.ts

@@ -315,7 +315,20 @@ export function buildApp(opts: AppOptions = {}) {
   const logAuditEvent = createAuditLogger(prisma as unknown as Parameters<typeof createAuditLogger>[0], fastify.log);
 
 // Setup plugins
-fastify.register(helmet, { contentSecurityPolicy: false, hsts: { maxAge: 31536000 }, referrerPolicy: { policy: 'no-referrer' } });
+fastify.register(helmet, {
+  contentSecurityPolicy: false,
+  crossOriginEmbedderPolicy: { policy: 'require-corp' },
+  crossOriginOpenerPolicy: { policy: 'same-origin' },
+  crossOriginResourcePolicy: { policy: 'same-origin' },
+  referrerPolicy: { policy: 'strict-origin-when-cross-origin' },
+  hsts: { maxAge: 31536000 },
+});
+
+fastify.addHook('onSend', async (_request, reply, _payload) => {
+  if (!reply.getHeader('permissions-policy')) {
+    reply.header('Permissions-Policy', 'geolocation=(), microphone=(), camera=()');
+  }
+});
 
 fastify.register(cors, {
   origin: env.ALLOWED_ORIGINS,

shared/validation/package.json

@@ -17,6 +17,7 @@
   },
   "dependencies": {
     "@bettapay/stellar-utils": "workspace:^",
+    "ioredis": "^5.11.1",
     "zod": "^3.22.4"
   }
 }

shared/validation/plugins.test.ts

@@ -2,7 +2,7 @@ import test from 'node:test';
 import assert from 'node:assert';
 import Fastify from 'fastify';
 import { z } from 'zod';
-import { registerErrorHandler, registerServiceAuth } from './plugins.js';
+import { registerErrorHandler, registerServiceAuth, redactPiiFromDetails } from './plugins.js';
 
 test('Zod validation error returns 400', async (t) => {
   const fastify = Fastify({ logger: false });
@@ -106,3 +106,72 @@ test('serviceAuth accepts a valid x-service-token', async () => {
   assert.strictEqual(res.statusCode, 200);
   assert.strictEqual(JSON.parse(res.body).ok, true);
 });
+
+test('redactPiiFromDetails redacts email field messages', () => {
+  const details = [{ path: ['email'], message: 'user@example.com is invalid' }];
+  const result = redactPiiFromDetails(details) as any;
+  assert.strictEqual(result[0].message, '[REDACTED]');
+});
+
+test('redactPiiFromDetails redacts secret key field messages', () => {
+  const details = [{ path: ['secretKey'], message: 'sk_live_abc123 is invalid' }];
+  const result = redactPiiFromDetails(details) as any;
+  assert.strictEqual(result[0].message, '[REDACTED]');
+});
+
+test('redactPiiFromDetails preserves non-PII field messages', () => {
+  const details = [{ path: ['amount'], message: 'Must be a positive number' }];
+  const result = redactPiiFromDetails(details) as any;
+  assert.strictEqual(result[0].message, 'Must be a positive number');
+});
+
+test('redactPiiFromDetails preserves error structure', () => {
+  const details = [
+    { path: ['email'], message: 'user@example.com is invalid' },
+    { path: ['amount'], message: 'Must be a positive number' },
+    { path: ['secretKey'], message: 'sk_live_abc123 is invalid' },
+  ];
+  const result = redactPiiFromDetails(details) as any;
+  assert.strictEqual(result[0].message, '[REDACTED]');
+  assert.strictEqual(result[1].message, 'Must be a positive number');
+  assert.strictEqual(result[2].message, '[REDACTED]');
+  assert.strictEqual(result.length, 3);
+});
+
+test('Zod error handler redacts PII from response', async (t) => {
+  const fastify = Fastify({ logger: false });
+  registerErrorHandler(fastify);
+  const schema = z.object({ email: z.string().email() });
+  fastify.post('/', (req, reply) => {
+    schema.parse(req.body);
+    reply.send({ ok: true });
+  });
+  const response = await fastify.inject({
+    method: 'POST',
+    url: '/',
+    payload: { email: 'not-an-email' }
+  });
+  assert.strictEqual(response.statusCode, 400);
+  const body = JSON.parse(response.body);
+  assert.strictEqual(body.error.details[0].message, '[REDACTED]');
+});
+
+test('error handler does not leak PII in message', async (t) => {
+  const fastify = Fastify({ logger: false });
+  registerErrorHandler(fastify);
+  fastify.post('/', (req, reply) => {
+    const schema = z.object({
+      email: z.string().refine(() => false, { message: 'user@example.com is invalid' })
+    });
+    schema.parse(req.body);
+    reply.send({ ok: true });
+  });
+  const response = await fastify.inject({
+    method: 'POST',
+    url: '/',
+    payload: { email: 'anything' }
+  });
+  const body = JSON.parse(response.body);
+  assert.strictEqual(body.error.details[0].message, '[REDACTED]');
+  assert.strictEqual(response.body.includes('user@example.com'), false);
+});

shared/validation/plugins.ts

@@ -61,20 +61,43 @@ export function classifyError(error: unknown, statusCode?: number): ErrorClass {
   return 'fatal';
 }
 
+export const PII_FIELD_PATTERNS = [/email/i, /address/i, /secret/i, /key/i, /token/i];
+
+export function isPiiField(path: (string | number)[]): boolean {
+  return path.some((segment) =>
+    typeof segment === 'string' && PII_FIELD_PATTERNS.some((re) => re.test(segment))
+  );
+}
+
+export function redactPiiFromDetails(details: unknown): unknown {
+  if (!Array.isArray(details)) return details;
+  return details.map((item: Record<string, unknown>) => {
+    const path: (string | number)[] = Array.isArray(item.path)
+      ? (item.path as (string | number)[])
+      : typeof item.instancePath === 'string'
+        ? item.instancePath.split('/').filter(Boolean)
+        : [];
+    if (isPiiField(path)) {
+      return { ...item, message: '[REDACTED]', received: undefined, data: undefined, value: undefined, params: undefined };
+    }
+    return item;
+  });
+}
+
 export function registerErrorHandler(fastify: FastifyInstance, customLogger?: FastifyBaseLogger) {
   fastify.setErrorHandler((error, request, reply) => {
     const logger = customLogger || request.log || fastify.log;
 
     if (error instanceof z.ZodError) {
-      const response = createErrorResponse(ErrorCodes.VALIDATION_ERROR, 'Invalid request data', error.errors);
+      const response = createErrorResponse(ErrorCodes.VALIDATION_ERROR, 'Invalid request data', redactPiiFromDetails(error.errors));
       return reply.code(400).send(response);
     }
 
     if ((error as FastifyError).statusCode) {
-      const fastifyErr = error as FastifyError;
-      // Use the attached status code. Preserve the safe message.
+      const fastifyErr = error as FastifyError & { validation?: unknown };
       const code = fastifyErr.code || ErrorCodes.INVALID_REQUEST;
-      const response = createErrorResponse(code, fastifyErr.message);
+      const details = fastifyErr.validation ? redactPiiFromDetails(fastifyErr.validation) : undefined;
+      const response = createErrorResponse(code, fastifyErr.message, details);
       return reply.code(fastifyErr.statusCode!).send(response);
     }
 

shared/validation/prisma.test.ts

@@ -146,3 +146,83 @@ test('buildPrismaConnectionUrl uses defaults when no poolSize or timeout given',
   const result = buildPrismaConnectionUrl(url);
   assert.strictEqual(result, 'postgresql://user:pass@localhost:5432/bettapay?connection_limit=10&pool_timeout=10');
 });
+
+import {
+  resetRotation,
+  hasRotated,
+  getActiveConnectionUrl,
+  connectWithRetryWithRotation,
+} from './prisma.js';
+
+test('connectWithRetryWithRotation switches to rotate URL on 28P01', async () => {
+  resetRotation();
+  let attempts = 0;
+  const prisma = {
+    async $connect() {
+      attempts += 1;
+      if (attempts === 1) {
+        const err = new Error('authentication failed');
+        (err as any).code = '28P01';
+        throw err;
+      }
+    },
+  };
+
+  await connectWithRetryWithRotation(prisma, {
+    debug: () => undefined,
+    warn: () => undefined,
+  }, { rotationUrl: 'postgres://rotated/db', baseDelayMs: 1, maxRetries: 5 });
+
+  assert.strictEqual(hasRotated(), true);
+  assert.strictEqual(getActiveConnectionUrl('postgres://primary/db'), 'postgres://rotated/db');
+  resetRotation();
+});
+
+test('connectWithRetryWithRotation does not switch on primary URL success', async () => {
+  resetRotation();
+  const prisma = {
+    async $connect() {},
+  };
+
+  await connectWithRetryWithRotation(prisma, {
+    debug: () => undefined,
+    warn: () => undefined,
+  }, { rotationUrl: 'postgres://rotated/db', baseDelayMs: 1, maxRetries: 3 });
+
+  assert.strictEqual(hasRotated(), false);
+  assert.strictEqual(getActiveConnectionUrl('postgres://primary/db'), 'postgres://primary/db');
+  resetRotation();
+});
+
+test('rotation logged at warn level', async () => {
+  resetRotation();
+  let attempts = 0;
+  const prisma = {
+    async $connect() {
+      attempts += 1;
+      if (attempts === 1) {
+        const err = new Error('authentication failed');
+        (err as any).code = '28P01';
+        throw err;
+      }
+    },
+  };
+
+  const warnMessages: string[] = [];
+  const logger = {
+    debug: () => undefined,
+    warn: (_obj: object, msg?: string) => {
+      if (msg) warnMessages.push(msg);
+    },
+  };
+
+  await connectWithRetryWithRotation(prisma, logger, { rotationUrl: 'postgres://rotated/db', baseDelayMs: 1, maxRetries: 5 });
+
+  assert.ok(warnMessages.some(m => m.includes('credential rotation') || m.includes('authentication error')));
+  resetRotation();
+});
+
+test('getActiveConnectionUrl returns primary URL when no rotation', () => {
+  resetRotation();
+  assert.strictEqual(getActiveConnectionUrl('postgres://primary/db'), 'postgres://primary/db');
+});

shared/validation/prisma.ts

@@ -2,6 +2,27 @@ export type PrismaLogLevel = 'query' | 'info' | 'warn' | 'error';
 
 const ALL_PRISMA_LOG_LEVELS: readonly PrismaLogLevel[] = ['query', 'info', 'warn', 'error'];
 
+let _rotateUrl: string | undefined;
+let _hasRotated = false;
+
+export function setRotationUrl(url: string | undefined): void {
+  _rotateUrl = url;
+  _hasRotated = false;
+}
+
+export function resetRotation(): void {
+  _rotateUrl = undefined;
+  _hasRotated = false;
+}
+
+export function hasRotated(): boolean {
+  return _hasRotated;
+}
+
+export function getActiveConnectionUrl(primaryUrl: string): string {
+  return _hasRotated && _rotateUrl !== undefined ? _rotateUrl : primaryUrl;
+}
+
 function isPrismaLogLevel(value: string): value is PrismaLogLevel {
   return (ALL_PRISMA_LOG_LEVELS as readonly string[]).includes(value);
 }
@@ -129,3 +150,53 @@ export function buildPrismaConnectionUrl(
   const sep = rawUrl.includes('?') ? '&' : '?';
   return `${rawUrl}${sep}connection_limit=${poolSize}&pool_timeout=${timeout}`;
 }
+
+export interface ConnectWithRotationOptions extends ConnectWithRetryOptions {
+  rotationUrl?: string;
+  logger?: PrismaLogger;
+}
+
+export async function connectWithRetryWithRotation(
+  prisma: PrismaConnectable,
+  logger: PrismaLogger,
+  options: ConnectWithRotationOptions = {}
+): Promise<void> {
+  const maxRetries = options.maxRetries ?? 10;
+  const baseDelayMs = options.baseDelayMs ?? 1000;
+  const maxDelayMs = options.maxDelayMs ?? 30000;
+  const rotationUrl = options.rotationUrl;
+  let lastError: unknown;
+
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    try {
+      await prisma.$connect();
+      return;
+    } catch (err) {
+      lastError = err;
+      const isAuthError = (err as { code?: string })?.code === '28P01';
+
+      if (isAuthError && rotationUrl && !_hasRotated) {
+        _hasRotated = true;
+        _rotateUrl = rotationUrl;
+        const rotationLogger = options.logger ?? logger;
+        rotationLogger.warn(
+          { attempt: attempt + 1 },
+          'Database credential rotation: switching to rotation URL due to authentication error (28P01)'
+        );
+      }
+
+      if (attempt < maxRetries - 1) {
+        const delay = Math.min(baseDelayMs * Math.pow(2, attempt), maxDelayMs);
+        logger.warn(
+          { attempt: attempt + 1, maxRetries, delayMs: delay, err },
+          'Database connection failed, retrying'
+        );
+        await sleep(delay);
+      }
+    }
+  }
+
+  const message =
+    lastError instanceof Error ? lastError.message : String(lastError ?? 'unknown error');
+  throw new Error(`Failed to connect to database after ${maxRetries} attempts: ${message}`);
+}