fix: properly clean-up when evaluation CLI is aborted

* Ensures we properly clean-up the evaluation CLI when user's abort
* Passes around the abort signals properly.

Author: devversion

runner/codegen/genkit/genkit-runner.ts

@@ -20,6 +20,7 @@ import {MODEL_PROVIDERS} from './models.js';
 import {UserFacingError} from '../../utils/errors.js';
 import {GenkitModelProvider, PromptDataForCounting} from './model-provider.js';
 import {ToolLogEntry} from '../../shared-interfaces.js';
+import {combineAbortSignals} from '../../utils/abort-signal.js';
 
 const globalLogger = new GenkitLogger();
 logger.init(globalLogger);
@@ -115,7 +116,7 @@ export class GenkitRunner implements LlmRunner {
       {messages: options.messages || [], prompt: options.prompt},
       () => {
         const schema = (options as Partial<LocalLlmConstrainedOutputGenerateRequestOptions>).schema;
-        const performRequest = async () => {
+        const performRequest = async (abortSignal: AbortSignal) => {
           let tools: ToolAction[] | undefined;
           let resources: DynamicResourceAction[] | undefined;
 
@@ -148,7 +149,7 @@ export class GenkitRunner implements LlmRunner {
             messages: options.messages,
             tools,
             resources,
-            abortSignal: options.abortSignal,
+            abortSignal,
           });
 
           this._logToolUsage(response);
@@ -159,10 +160,11 @@ export class GenkitRunner implements LlmRunner {
         return options.timeout
           ? callWithTimeout(
               options.timeout.description,
-              performRequest,
+              timeoutAbortSignal =>
+                performRequest(combineAbortSignals(timeoutAbortSignal, options.abortSignal)),
               options.timeout.durationInMins,
             )
-          : performRequest();
+          : performRequest(options.abortSignal);
       },
     );
   }

runner/eval-cli.ts

@@ -190,6 +190,13 @@ async function handler(cliArgs: Arguments<Options>): Promise<void> {
     process.exit(0);
   }
 
+  const abortCtrl = new AbortController();
+
+  process.on('SIGINT', () => abortCtrl.abort());
+  process.on('SIGTERM', () => abortCtrl.abort());
+  process.on('SIGTERM', () => abortCtrl.abort());
+  process.on('exit', () => abortCtrl.abort());
+
   try {
     const runInfo = await generateCodeAndAssess({
       runner: cliArgs.runner,
@@ -214,6 +221,7 @@ async function handler(cliArgs: Arguments<Options>): Promise<void> {
       skipLighthouse: cliArgs.skipLighthouse,
       maxBuildRepairAttempts: cliArgs.maxBuildRepairAttempts,
       maxTestRepairAttempts: cliArgs.maxTestRepairAttempts,
+      abortSignal: abortCtrl.signal,
     });
 
     logReportToConsole(runInfo);

runner/orchestration/executors/local-executor.ts

@@ -26,6 +26,7 @@ import {getPossiblePackageManagers} from '../../configuration/package-managers.j
 import {callWithTimeout} from '../../utils/timeout.js';
 import {executeCommand} from '../../utils/exec.js';
 import {cleanupBuildMessage} from '../../workers/builder/worker.js';
+import {combineAbortSignals} from '../../utils/abort-signal.js';
 
 let uniqueIDs = 0;
 
@@ -145,7 +146,7 @@ export class LocalExecutor implements Executor {
           `Testing ${rootPromptDef.name}`,
           timeoutAbort =>
             executeCommand(testCommand, appDirectoryPath, undefined, {
-              abortSignal: AbortSignal.any([abortSignal, timeoutAbort]),
+              abortSignal: combineAbortSignals(abortSignal, timeoutAbort),
             }),
           4, // 4min. This is a safety boundary. Lots of parallelism can slow-down.
         ),

runner/orchestration/generate.ts

@@ -26,6 +26,7 @@ import {LocalExecutor} from './executors/local-executor.js';
 import {startEvaluationTask} from './generate-eval-task.js';
 import {prepareSummary} from './generate-summary.js';
 import {getRunGroupId} from './grouping.js';
+import {combineAbortSignals} from '../utils/abort-signal.js';
 
 /**
  * Orchestrates the entire assessment process for each prompt defined in the `prompts` array.
@@ -42,6 +43,13 @@ import {getRunGroupId} from './grouping.js';
  */
 export async function generateCodeAndAssess(options: AssessmentConfig): Promise<RunInfo> {
   const env = await getEnvironmentByPath(options.environmentConfigPath, options.runner);
+  const cleanup = async () => {
+    await env.executor.destroy();
+  };
+
+  // Ensure cleanup logic runs when the evaluation is aborted.
+  options.abortSignal?.addEventListener('abort', cleanup);
+
   const ratingLlm = await getRunnerByName('genkit');
 
   await assertValidModelName(options.model, env.executor);
@@ -106,14 +114,14 @@ export async function generateCodeAndAssess(options: AssessmentConfig): Promise<
           try {
             results = await callWithTimeout(
               `Evaluation of ${rootPromptDef.name}`,
-              async abortSignal =>
+              async timeoutAbortSignal =>
                 startEvaluationTask(
                   options,
                   evalID,
                   env,
                   ratingLlm,
                   rootPromptDef,
-                  abortSignal,
+                  combineAbortSignals(timeoutAbortSignal, options.abortSignal),
                   workerConcurrencyQueue,
                   progress,
                 ),
@@ -189,7 +197,10 @@ export async function generateCodeAndAssess(options: AssessmentConfig): Promise<
       details,
     } satisfies RunInfo;
   } finally {
-    await env.executor.destroy();
+    await cleanup();
+
+    // Remove potential abort listeners to avoid memory leaks.
+    options.abortSignal?.removeEventListener('abort', cleanup);
   }
 }
 

runner/shared-interfaces.ts

@@ -30,6 +30,7 @@ export interface AssessmentConfig {
   skipLighthouse?: boolean;
   maxTestRepairAttempts?: number;
   maxBuildRepairAttempts?: number;
+  abortSignal?: AbortSignal;
 }
 
 /**

runner/utils/abort-signal.ts

@@ -0,0 +1,3 @@
+export function combineAbortSignals(...signals: (AbortSignal | undefined)[]): AbortSignal {
+  return AbortSignal.any(signals.filter(s => s !== undefined));
+}