fix(health): force alert notifications on health command runs

Author: utkarsh232005

src/__tests__/alerts.test.ts

@@ -0,0 +1,96 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { triggerAlert } from '../monitor/alerts';
+import { logger } from '../utils/logger';
+import { getConfig } from '../utils/config';
+
+// Mock config
+vi.mock('../utils/config', () => ({
+  getConfig: vi.fn(() => ({
+    alert_cooldown: 300,
+    discord_webhook: 'https://discord.com/api/webhooks/dummy',
+  })),
+  getSMTPSettings: vi.fn(() => ({
+    host: '',
+    port: 587,
+    auth: { user: '', pass: '' },
+    to: '',
+  })),
+}));
+
+// Mock logger
+vi.mock('../utils/logger', () => ({
+  logger: {
+    info: vi.fn(),
+    error: vi.fn(),
+  },
+}));
+
+// Mock fetch globally
+const mockFetch = vi.fn().mockResolvedValue({ ok: true });
+global.fetch = mockFetch;
+
+describe('alerts monitoring', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should trigger alert on first invocation', async () => {
+    const alertId = 'container:test-first:failure';
+    await triggerAlert({
+      id: alertId,
+      message: 'Container crashed',
+      type: 'container',
+      severity: 'critical',
+    });
+
+    expect(logger.info).toHaveBeenCalledWith(expect.stringContaining(`Triggering alert for ${alertId}`));
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+  });
+
+  it('should respect cooldown and suppress alert on subsequent invocations within cooldown window', async () => {
+    const alertId = 'container:test-cooldown:failure';
+    
+    // First trigger - should send
+    await triggerAlert({
+      id: alertId,
+      message: 'Container crashed first time',
+      type: 'container',
+      severity: 'critical',
+    });
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+    
+    // Second trigger within cooldown - should be suppressed
+    await triggerAlert({
+      id: alertId,
+      message: 'Container crashed second time',
+      type: 'container',
+      severity: 'critical',
+    });
+    // Call count should still be 1
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+  });
+
+  it('should bypass cooldown and trigger alert on subsequent invocations if force option is true', async () => {
+    const alertId = 'container:test-force:failure';
+    
+    // First trigger - should send
+    await triggerAlert({
+      id: alertId,
+      message: 'Container crashed first time',
+      type: 'container',
+      severity: 'critical',
+    });
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+    
+    // Second trigger with force: true - should send regardless of cooldown
+    await triggerAlert({
+      id: alertId,
+      message: 'Container crashed second time',
+      type: 'container',
+      severity: 'critical',
+    }, { force: true });
+    
+    // Call count should be 2 now
+    expect(mockFetch).toHaveBeenCalledTimes(2);
+  });
+});

src/__tests__/health.test.ts

@@ -85,6 +85,8 @@ describe('health command', () => {
 
     await program.parseAsync(['node', 'test', 'health', 'all']);
 
+    expect(getRunningContainers).toHaveBeenCalledWith({ forceAlert: true });
+    expect(getRunningPods).toHaveBeenCalledWith({ forceAlert: true });
     expect(logger.info).toHaveBeenCalledWith('Showing health for all...');
     expect(tableUtils.renderTable).toHaveBeenCalledWith(
       expect.objectContaining({

src/commands/health.ts

@@ -48,14 +48,14 @@ const healthColor = (status: string): string => {
   return chalk.yellow(status);
 };
 
-const fetchHealthRows = async (target: string): Promise<(string | number)[][]> => {
+const fetchHealthRows = async (target: string, options?: { forceAlert?: boolean }): Promise<(string | number)[][]> => {
   const rows: (string | number)[][] = [];
   const shouldFetchContainers = target === 'all' || target === 'containers';
   const shouldFetchPods = target === 'all' || target === 'pods';
 
   const [containerResult, podResult] = await Promise.allSettled([
-    shouldFetchContainers ? getRunningContainers() : Promise.resolve([]),
-    shouldFetchPods ? getRunningPods() : Promise.resolve([]),
+    shouldFetchContainers ? getRunningContainers({ forceAlert: options?.forceAlert }) : Promise.resolve([]),
+    shouldFetchPods ? getRunningPods({ forceAlert: options?.forceAlert }) : Promise.resolve([]),
   ]);
 
   if (shouldFetchContainers) {
@@ -150,7 +150,7 @@ export const showHealth = async (target: string, options: HealthOptions = {}): P
         return;
       }
 
-      const rows = await fetchHealthRows(target);
+      const rows = await fetchHealthRows(target, { forceAlert: true });
 
       // Clear terminal screen
       process.stdout.write('\x1Bc');
@@ -180,7 +180,7 @@ export const showHealth = async (target: string, options: HealthOptions = {}): P
     await poll();
   } else {
     const spinner = createSpinner(`Checking ${target} health...`).start();
-    const rows = await fetchHealthRows(target);
+    const rows = await fetchHealthRows(target, { forceAlert: true });
     spinner.stop();
 
     if (rows.length === 0) {

src/docker/containers.ts

@@ -10,7 +10,7 @@ export interface ContainerData {
   status: string;
 }
 
-export const getRunningContainers = async (): Promise<ContainerData[]> => {
+export const getRunningContainers = async (options?: { forceAlert?: boolean }): Promise<ContainerData[]> => {
   const docker = getDockerClient();
   try {
     // Try to list containers, use a timeout if possible or just catch common connection errors
@@ -27,7 +27,7 @@ export const getRunningContainers = async (): Promise<ContainerData[]> => {
           type: 'container',
           severity: 'warning',
           message: `Docker container ${name} (${id}) is restarting.`,
-        });
+        }, { force: options?.forceAlert });
       } else if (c.State === 'exited') {
         const match = c.Status.match(/Exited \((\d+)\)/);
         const exitCode = match ? parseInt(match[1], 10) : 0;
@@ -38,7 +38,7 @@ export const getRunningContainers = async (): Promise<ContainerData[]> => {
             type: 'container',
             severity: 'critical',
             message: `Docker container ${name} (${id}) exited with code ${exitCode}.`,
-          });
+          }, { force: options?.forceAlert });
         }
       }
 

src/kubernetes/pods.ts

@@ -11,7 +11,7 @@ export interface PodData {
   node: string;
 }
 
-export const getRunningPods = async (): Promise<PodData[]> => {
+export const getRunningPods = async (options?: { forceAlert?: boolean }): Promise<PodData[]> => {
   const api = getK8sApi();
   try {
     const res = await api.listPodForAllNamespaces();
@@ -43,7 +43,7 @@ export const getRunningPods = async (): Promise<PodData[]> => {
           type: 'pod',
           severity: 'critical',
           message: `Pod ${name} in namespace ${pod.metadata?.namespace} failed: ${failureReason}`,
-        });
+        }, { force: options?.forceAlert });
       }
 
       return {

src/monitor/alerts.ts

@@ -70,13 +70,13 @@ const sendEmailNotification = async (alert: Alert) => {
   }
 };
 
-export const triggerAlert = async (alert: Alert) => {
+export const triggerAlert = async (alert: Alert, options?: { force?: boolean }) => {
   const now = Date.now();
   const { alert_cooldown = 300 } = getConfig(); // Default 5 minutes
   const cooldownMs = alert_cooldown * 1000;
 
   const lastAlert = cooldownTracker.get(alert.id);
-  if (lastAlert && (now - lastAlert) < cooldownMs) {
+  if (!options?.force && lastAlert && (now - lastAlert) < cooldownMs) {
     return; // Still in cooldown
   }