From c4b73722ba8ba8ec348bbde3983582e97cd353ff Mon Sep 17 00:00:00 2001 From: Chuan-kai Lin Date: Wed, 24 Sep 2025 11:43:34 -0700 Subject: [PATCH 1/4] Add overlay-base database cache key tests --- src/overlay-database-utils.test.ts | 39 ++++++++++++++++++++++++++++++ src/overlay-database-utils.ts | 4 +-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/overlay-database-utils.test.ts b/src/overlay-database-utils.test.ts index ca52f1d88a..d8a7c05bda 100644 --- a/src/overlay-database-utils.test.ts +++ b/src/overlay-database-utils.test.ts @@ -11,6 +11,8 @@ import * as gitUtils from "./git-utils"; import { getRunnerLogger } from "./logging"; import { downloadOverlayBaseDatabaseFromCache, + getCacheRestoreKeyPrefix, + getCacheSaveKey, OverlayDatabaseMode, writeBaseDatabaseOidsFile, writeOverlayChangesFile, @@ -261,3 +263,40 @@ test( }, false, ); + +test("overlay-base database cache keys remain stable", async (t) => { + const config = createTestConfig({ languages: ["python", "javascript"] }); + const codeQlVersion = "2.23.0"; + const commitOid = "abc123def456"; + + sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/"); + sinon.stub(gitUtils, "getCommitOid").resolves(commitOid); + + const saveKey = await getCacheSaveKey(config, codeQlVersion, "checkout-path"); + const expectedSaveKey = + "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-abc123def456"; + t.is( + saveKey, + expectedSaveKey, + "Cache save key changed unexpectedly. " + + "This may indicate breaking changes in the cache key generation logic.", + ); + + const restoreKeyPrefix = await getCacheRestoreKeyPrefix( + config, + codeQlVersion, + ); + const expectedRestoreKeyPrefix = + "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-"; + t.is( + restoreKeyPrefix, + expectedRestoreKeyPrefix, + "Cache restore key prefix changed unexpectedly. " + + "This may indicate breaking changes in the cache key generation logic.", + ); + + t.true( + saveKey.startsWith(restoreKeyPrefix), + `Expected save key "${saveKey}" to start with restore key prefix "${restoreKeyPrefix}"`, + ); +}); diff --git a/src/overlay-database-utils.ts b/src/overlay-database-utils.ts index 1de76fef77..81d39a990f 100644 --- a/src/overlay-database-utils.ts +++ b/src/overlay-database-utils.ts @@ -448,7 +448,7 @@ export async function downloadOverlayBaseDatabaseFromCache( * The key consists of the restore key prefix (which does not include the * commit SHA) and the commit SHA of the current checkout. */ -async function getCacheSaveKey( +export async function getCacheSaveKey( config: Config, codeQlVersion: string, checkoutPath: string, @@ -475,7 +475,7 @@ async function getCacheSaveKey( * not include the commit SHA. This allows us to restore the most recent * compatible overlay-base database. */ -async function getCacheRestoreKeyPrefix( +export async function getCacheRestoreKeyPrefix( config: Config, codeQlVersion: string, ): Promise { From b4ce3352864e24176b5a5926124bb6adfc45b003 Mon Sep 17 00:00:00 2001 From: Kasper Svendsen Date: Fri, 24 Oct 2025 10:48:01 +0200 Subject: [PATCH 2/4] Ensure uniqueness of overlay-base database cache keys --- lib/analyze-action.js | 4 +++- src/overlay-database-utils.test.ts | 4 +++- src/overlay-database-utils.ts | 11 +++++++++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/analyze-action.js b/lib/analyze-action.js index 8b936b19d2..9efff648cb 100644 --- a/lib/analyze-action.js +++ b/lib/analyze-action.js @@ -91075,12 +91075,14 @@ async function uploadOverlayBaseDatabaseToCache(codeql, config, logger) { return true; } async function getCacheSaveKey(config, codeQlVersion, checkoutPath) { + const runId = getWorkflowRunID(); + const attemptId = getWorkflowRunAttempt(); const sha = await getCommitOid(checkoutPath); const restoreKeyPrefix = await getCacheRestoreKeyPrefix( config, codeQlVersion ); - return `${restoreKeyPrefix}${sha}`; + return `${restoreKeyPrefix}${runId}-${attemptId}-${sha}`; } async function getCacheRestoreKeyPrefix(config, codeQlVersion) { const languages = [...config.languages].sort().join("_"); diff --git a/src/overlay-database-utils.test.ts b/src/overlay-database-utils.test.ts index d8a7c05bda..7124d4ece4 100644 --- a/src/overlay-database-utils.test.ts +++ b/src/overlay-database-utils.test.ts @@ -271,10 +271,12 @@ test("overlay-base database cache keys remain stable", async (t) => { sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/"); sinon.stub(gitUtils, "getCommitOid").resolves(commitOid); + sinon.stub(actionsUtil, "getWorkflowRunID").returns(12345); + sinon.stub(actionsUtil, "getWorkflowRunAttempt").returns(1); const saveKey = await getCacheSaveKey(config, codeQlVersion, "checkout-path"); const expectedSaveKey = - "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-abc123def456"; + "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-12345-1-abc123def456"; t.is( saveKey, expectedSaveKey, diff --git a/src/overlay-database-utils.ts b/src/overlay-database-utils.ts index 81d39a990f..e3d06866f8 100644 --- a/src/overlay-database-utils.ts +++ b/src/overlay-database-utils.ts @@ -4,7 +4,12 @@ import * as path from "path"; import * as actionsCache from "@actions/cache"; -import { getRequiredInput, getTemporaryDirectory } from "./actions-util"; +import { + getRequiredInput, + getTemporaryDirectory, + getWorkflowRunAttempt, + getWorkflowRunID, +} from "./actions-util"; import { getAutomationID } from "./api-client"; import { type CodeQL } from "./codeql"; import { type Config } from "./config-utils"; @@ -453,12 +458,14 @@ export async function getCacheSaveKey( codeQlVersion: string, checkoutPath: string, ): Promise { + const runId = getWorkflowRunID(); + const attemptId = getWorkflowRunAttempt(); const sha = await getCommitOid(checkoutPath); const restoreKeyPrefix = await getCacheRestoreKeyPrefix( config, codeQlVersion, ); - return `${restoreKeyPrefix}${sha}`; + return `${restoreKeyPrefix}${runId}-${attemptId}-${sha}`; } /** From cbcae45fffacea44eec0222c515b06a260497ab4 Mon Sep 17 00:00:00 2001 From: Kasper Svendsen Date: Fri, 24 Oct 2025 15:46:17 +0200 Subject: [PATCH 3/4] Reorder components of overlay-base cache key postfix --- lib/analyze-action.js | 2 +- src/overlay-database-utils.test.ts | 2 +- src/overlay-database-utils.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/analyze-action.js b/lib/analyze-action.js index 9efff648cb..bcb46d5d3e 100644 --- a/lib/analyze-action.js +++ b/lib/analyze-action.js @@ -91082,7 +91082,7 @@ async function getCacheSaveKey(config, codeQlVersion, checkoutPath) { config, codeQlVersion ); - return `${restoreKeyPrefix}${runId}-${attemptId}-${sha}`; + return `${restoreKeyPrefix}${sha}-${runId}-${attemptId}`; } async function getCacheRestoreKeyPrefix(config, codeQlVersion) { const languages = [...config.languages].sort().join("_"); diff --git a/src/overlay-database-utils.test.ts b/src/overlay-database-utils.test.ts index 7124d4ece4..d04668e980 100644 --- a/src/overlay-database-utils.test.ts +++ b/src/overlay-database-utils.test.ts @@ -276,7 +276,7 @@ test("overlay-base database cache keys remain stable", async (t) => { const saveKey = await getCacheSaveKey(config, codeQlVersion, "checkout-path"); const expectedSaveKey = - "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-12345-1-abc123def456"; + "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-abc123def456-12345-1"; t.is( saveKey, expectedSaveKey, diff --git a/src/overlay-database-utils.ts b/src/overlay-database-utils.ts index e3d06866f8..bb62ec0c90 100644 --- a/src/overlay-database-utils.ts +++ b/src/overlay-database-utils.ts @@ -465,7 +465,7 @@ export async function getCacheSaveKey( config, codeQlVersion, ); - return `${restoreKeyPrefix}${runId}-${attemptId}-${sha}`; + return `${restoreKeyPrefix}${sha}-${runId}-${attemptId}`; } /** From 66759e57b29d9322a4b80dadf4ab043a9531b4cd Mon Sep 17 00:00:00 2001 From: Kasper Svendsen Date: Fri, 24 Oct 2025 15:48:42 +0200 Subject: [PATCH 4/4] Improve error handling for overlay-base cache key creation --- lib/analyze-action.js | 17 +++++++++++++---- src/overlay-database-utils.test.ts | 8 +++++++- src/overlay-database-utils.ts | 15 +++++++++++++-- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/lib/analyze-action.js b/lib/analyze-action.js index bcb46d5d3e..23e4d2c47b 100644 --- a/lib/analyze-action.js +++ b/lib/analyze-action.js @@ -91049,7 +91049,8 @@ async function uploadOverlayBaseDatabaseToCache(codeql, config, logger) { const cacheSaveKey = await getCacheSaveKey( config, codeQlVersion, - checkoutPath + checkoutPath, + logger ); logger.info( `Uploading overlay-base database to Actions cache with key ${cacheSaveKey}` @@ -91074,9 +91075,17 @@ async function uploadOverlayBaseDatabaseToCache(codeql, config, logger) { logger.info(`Successfully uploaded overlay-base database from ${dbLocation}`); return true; } -async function getCacheSaveKey(config, codeQlVersion, checkoutPath) { - const runId = getWorkflowRunID(); - const attemptId = getWorkflowRunAttempt(); +async function getCacheSaveKey(config, codeQlVersion, checkoutPath, logger) { + let runId = 1; + let attemptId = 1; + try { + runId = getWorkflowRunID(); + attemptId = getWorkflowRunAttempt(); + } catch (e) { + logger.warning( + `Failed to get workflow run ID or attempt ID. Reason: ${getErrorMessage(e)}` + ); + } const sha = await getCommitOid(checkoutPath); const restoreKeyPrefix = await getCacheRestoreKeyPrefix( config, diff --git a/src/overlay-database-utils.test.ts b/src/overlay-database-utils.test.ts index d04668e980..cee0d45f13 100644 --- a/src/overlay-database-utils.test.ts +++ b/src/overlay-database-utils.test.ts @@ -265,6 +265,7 @@ test( ); test("overlay-base database cache keys remain stable", async (t) => { + const logger = getRunnerLogger(true); const config = createTestConfig({ languages: ["python", "javascript"] }); const codeQlVersion = "2.23.0"; const commitOid = "abc123def456"; @@ -274,7 +275,12 @@ test("overlay-base database cache keys remain stable", async (t) => { sinon.stub(actionsUtil, "getWorkflowRunID").returns(12345); sinon.stub(actionsUtil, "getWorkflowRunAttempt").returns(1); - const saveKey = await getCacheSaveKey(config, codeQlVersion, "checkout-path"); + const saveKey = await getCacheSaveKey( + config, + codeQlVersion, + "checkout-path", + logger, + ); const expectedSaveKey = "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-abc123def456-12345-1"; t.is( diff --git a/src/overlay-database-utils.ts b/src/overlay-database-utils.ts index bb62ec0c90..7934c50140 100644 --- a/src/overlay-database-utils.ts +++ b/src/overlay-database-utils.ts @@ -16,6 +16,7 @@ import { type Config } from "./config-utils"; import { getCommitOid, getFileOidsUnderPath } from "./git-utils"; import { Logger, withGroupAsync } from "./logging"; import { + getErrorMessage, isInTestMode, tryGetFolderBytes, waitForResultWithTimeLimit, @@ -276,6 +277,7 @@ export async function uploadOverlayBaseDatabaseToCache( config, codeQlVersion, checkoutPath, + logger, ); logger.info( `Uploading overlay-base database to Actions cache with key ${cacheSaveKey}`, @@ -457,9 +459,18 @@ export async function getCacheSaveKey( config: Config, codeQlVersion: string, checkoutPath: string, + logger: Logger, ): Promise { - const runId = getWorkflowRunID(); - const attemptId = getWorkflowRunAttempt(); + let runId = 1; + let attemptId = 1; + try { + runId = getWorkflowRunID(); + attemptId = getWorkflowRunAttempt(); + } catch (e) { + logger.warning( + `Failed to get workflow run ID or attempt ID. Reason: ${getErrorMessage(e)}`, + ); + } const sha = await getCommitOid(checkoutPath); const restoreKeyPrefix = await getCacheRestoreKeyPrefix( config,