feat(cli): deterministic image builds for deployments (#2778)

myftija · web-flow · commit 6d6ed471d15f · 2025-12-12T09:48:04.000+01:00
This PR makes our image builds deterministic and reproducible by
ensuring that identical source code always produces the same image
layers and image digest. This means that deployments where nothing has
changed will no longer invalidate the image cache in our worker cluster
nodes, thus avoid making the cold starts for runs worse.

**Context**
New deployments currently increase the cold start times for runs, as
they generate a new image which needs to be pulled in the worker cluster
where runs are executed. It happens also when the source code for the
deployment has not changed due to non-deterministic steps in our build
system. This addresses the latter issue by making builds reproducible.

**Main changes**
- Avoided baking `TRIGGER_DEPLOYMENT_ID` and
`TRIGGER_DEPLOYMENT_VERSION` in the image, we now pass these via the
supervisor instead.
- Used `json-stable-stringify` for consistent key ordering in the files
we generate for the build, e.g., `package.json`, `build.json`,
`index.json`.
- Removed `metafile.json` from the image contents as it is not actually
used in the container. This is only relevant for the `analyze` command.
- Added `SOURCE_DATE_EPOCH=0` and `rewrite-timestamp=true` to Docker
builds to normalize file timestamps.
- Removed some `timings` and `outputHashes` from build outputs and
manifests.

The builds are now reproducible for both native build server and Depot
paths. This should also lead to better image layer cache reuse in
general.
diff --git a/.changeset/five-pens-fly.md b/.changeset/five-pens-fly.md
@@ -0,0 +1,6 @@
+---
+"trigger.dev": minor
+"@trigger.dev/core": minor
+---
+
+feat(cli): deterministic image builds for deployments
diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts
@@ -244,6 +244,12 @@ class ManagedSupervisor {
       }
 
       try {
+        if (!message.deployment.friendlyId) {
+          // mostly a type guard, deployments always exists for deployed environments
+          // a proper fix would be to use a discriminated union schema to differentiate between dequeued runs in dev and in deployed environments.
+          throw new Error("Deployment is missing");
+        }
+
         await this.workloadManager.create({
           dequeuedAt: message.dequeuedAt,
           envId: message.environment.id,
@@ -252,6 +258,8 @@ class ManagedSupervisor {
           machine: message.run.machine,
           orgId: message.organization.id,
           projectId: message.project.id,
+          deploymentFriendlyId: message.deployment.friendlyId,
+          deploymentVersion: message.backgroundWorker.version,
           runId: message.run.id,
           runFriendlyId: message.run.friendlyId,
           version: message.version,
diff --git a/apps/supervisor/src/workloadManager/docker.ts b/apps/supervisor/src/workloadManager/docker.ts
@@ -72,6 +72,8 @@ export class DockerWorkloadManager implements WorkloadManager {
       `TRIGGER_DEQUEUED_AT_MS=${opts.dequeuedAt.getTime()}`,
       `TRIGGER_POD_SCHEDULED_AT_MS=${Date.now()}`,
       `TRIGGER_ENV_ID=${opts.envId}`,
+      `TRIGGER_DEPLOYMENT_ID=${opts.deploymentFriendlyId}`,
+      `TRIGGER_DEPLOYMENT_VERSION=${opts.deploymentVersion}`,
       `TRIGGER_RUN_ID=${opts.runFriendlyId}`,
       `TRIGGER_SNAPSHOT_ID=${opts.snapshotFriendlyId}`,
       `TRIGGER_SUPERVISOR_API_PROTOCOL=${this.opts.workloadApiProtocol}`,
diff --git a/apps/supervisor/src/workloadManager/kubernetes.ts b/apps/supervisor/src/workloadManager/kubernetes.ts
@@ -123,6 +123,14 @@ export class KubernetesWorkloadManager implements WorkloadManager {
                     name: "TRIGGER_ENV_ID",
                     value: opts.envId,
                   },
+                  {
+                    name: "TRIGGER_DEPLOYMENT_ID",
+                    value: opts.deploymentFriendlyId,
+                  },
+                  {
+                    name: "TRIGGER_DEPLOYMENT_VERSION",
+                    value: opts.deploymentVersion,
+                  },
                   {
                     name: "TRIGGER_SNAPSHOT_ID",
                     value: opts.snapshotFriendlyId,
diff --git a/apps/supervisor/src/workloadManager/types.ts b/apps/supervisor/src/workloadManager/types.ts
@@ -29,6 +29,8 @@ export interface WorkloadManagerCreateOptions {
   envType: EnvironmentType;
   orgId: string;
   projectId: string;
+  deploymentFriendlyId: string;
+  deploymentVersion: string;
   runId: string;
   runFriendlyId: string;
   snapshotId: string;
diff --git a/internal-packages/run-engine/src/engine/systems/dequeueSystem.ts b/internal-packages/run-engine/src/engine/systems/dequeueSystem.ts
@@ -559,6 +559,8 @@ export class DequeueSystem {
                   friendlyId: result.worker.friendlyId,
                   version: result.worker.version,
                 },
+                // TODO: use a discriminated union schema to differentiate between dequeued runs in dev and in deployed environments.
+                // Would help make the typechecking stricter
                 deployment: {
                   id: result.deployment?.id,
                   friendlyId: result.deployment?.friendlyId,
diff --git a/packages/cli-v3/package.json b/packages/cli-v3/package.json
@@ -92,10 +92,10 @@
     "@opentelemetry/resources": "2.0.1",
     "@opentelemetry/sdk-trace-node": "2.0.1",
     "@opentelemetry/semantic-conventions": "1.36.0",
+    "@s2-dev/streamstore": "^0.17.6",
     "@trigger.dev/build": "workspace:4.2.0",
     "@trigger.dev/core": "workspace:4.2.0",
     "@trigger.dev/schema-to-json": "workspace:4.2.0",
-    "@s2-dev/streamstore": "^0.17.6",
     "ansi-escapes": "^7.0.0",
     "braces": "^3.0.3",
     "c12": "^1.11.1",
@@ -117,6 +117,7 @@
     "import-in-the-middle": "1.11.0",
     "import-meta-resolve": "^4.1.0",
     "ini": "^5.0.0",
+    "json-stable-stringify": "^1.3.0",
     "jsonc-parser": "3.2.1",
     "magicast": "^0.3.4",
     "minimatch": "^10.0.1",
diff --git a/packages/cli-v3/src/build/buildWorker.ts b/packages/cli-v3/src/build/buildWorker.ts
@@ -12,7 +12,7 @@ import { join, relative, sep } from "node:path";
 import { generateContainerfile } from "../deploy/buildImage.js";
 import { writeFile } from "node:fs/promises";
 import { buildManifestToJSON } from "../utilities/buildManifest.js";
-import { readPackageJSON, writePackageJSON } from "pkg-types";
+import { readPackageJSON } from "pkg-types";
 import { writeJSONFile } from "../utilities/fileSystem.js";
 import { isWindows } from "std-env";
 import { pathToFileURL } from "node:url";
@@ -192,20 +192,23 @@ async function writeDeployFiles({
     ) ?? {};
 
   // Step 3: Write the resolved dependencies to the package.json file
-  await writePackageJSON(join(outputPath, "package.json"), {
-    ...packageJson,
-    name: packageJson.name ?? "trigger-project",
-    dependencies: {
-      ...dependencies,
+  await writeJSONFile(
+    join(outputPath, "package.json"),
+    {
+      ...packageJson,
+      name: packageJson.name ?? "trigger-project",
+      dependencies: {
+        ...dependencies,
+      },
+      trustedDependencies: Object.keys(dependencies).sort(),
+      devDependencies: {},
+      peerDependencies: {},
+      scripts: {},
     },
-    trustedDependencies: Object.keys(dependencies),
-    devDependencies: {},
-    peerDependencies: {},
-    scripts: {},
-  });
+    true
+  );
 
   await writeJSONFile(join(outputPath, "build.json"), buildManifestToJSON(buildManifest));
-  await writeJSONFile(join(outputPath, "metafile.json"), bundleResult.metafile);
   await writeContainerfile(outputPath, buildManifest);
 }
 
diff --git a/packages/cli-v3/src/build/bundle.ts b/packages/cli-v3/src/build/bundle.ts
@@ -414,7 +414,9 @@ export async function createBuildManifestFromBundle({
     otelImportHook: {
       include: resolvedConfig.instrumentedPackageNames ?? [],
     },
-    outputHashes: bundle.outputHashes,
+    // `outputHashes` is only needed for dev builds for the deduplication mechanism during rebuilds.
+    // For deploys builds, we omit it to ensure deterministic builds
+    outputHashes: target === "dev" ? bundle.outputHashes : {},
   };
 
   if (!workerDir) {
diff --git a/packages/cli-v3/src/deploy/buildImage.ts b/packages/cli-v3/src/deploy/buildImage.ts
@@ -201,6 +201,7 @@ async function remoteBuildImage(options: DepotBuildImageOptions): Promise<BuildI
   const outputOptions = getOutputOptions({
     imageTag: undefined, // This is already handled via the --save flag
     push: true, // We always push the image to the registry
+    load: options.load,
     compression: options.compression,
     compressionLevel: options.compressionLevel,
     forceCompression: options.forceCompression,
@@ -213,18 +214,17 @@ async function remoteBuildImage(options: DepotBuildImageOptions): Promise<BuildI
     options.noCache ? "--no-cache" : undefined,
     "--platform",
     options.imagePlatform,
-    options.load ? "--load" : undefined,
     "--provenance",
     "false",
     "--metadata-file",
     "metadata.json",
     "--build-arg",
+    `SOURCE_DATE_EPOCH=0`,
+    "--build-arg",
     `TRIGGER_PROJECT_ID=${options.projectId}`,
     "--build-arg",
     `TRIGGER_DEPLOYMENT_ID=${options.deploymentId}`,
     "--build-arg",
-    `TRIGGER_DEPLOYMENT_VERSION=${options.deploymentVersion}`,
-    "--build-arg",
     `TRIGGER_CONTENT_HASH=${options.contentHash}`,
     "--build-arg",
     `TRIGGER_PROJECT_REF=${options.projectRef}`,
@@ -534,6 +534,7 @@ async function localBuildImage(options: SelfHostedBuildImageOptions): Promise<Bu
   const outputOptions = getOutputOptions({
     imageTag,
     push,
+    load,
     compression,
     compressionLevel,
     forceCompression,
@@ -563,18 +564,17 @@ async function localBuildImage(options: SelfHostedBuildImageOptions): Promise<Bu
     options.imagePlatform,
     options.network ? `--network=${options.network}` : undefined,
     addHost ? `--add-host=${addHost}` : undefined,
-    load ? "--load" : undefined,
     "--provenance",
     "false",
     "--metadata-file",
     "metadata.json",
     "--build-arg",
+    `SOURCE_DATE_EPOCH=0`,
+    "--build-arg",
     `TRIGGER_PROJECT_ID=${options.projectId}`,
     "--build-arg",
     `TRIGGER_DEPLOYMENT_ID=${options.deploymentId}`,
     "--build-arg",
-    `TRIGGER_DEPLOYMENT_VERSION=${options.deploymentVersion}`,
-    "--build-arg",
     `TRIGGER_CONTENT_HASH=${options.contentHash}`,
     "--build-arg",
     `TRIGGER_PROJECT_REF=${options.projectRef}`,
@@ -588,8 +588,6 @@ async function localBuildImage(options: SelfHostedBuildImageOptions): Promise<Bu
     ...(options.extraCACerts ? ["--build-arg", `NODE_EXTRA_CA_CERTS=${options.extraCACerts}`] : []),
     "--progress",
     "plain",
-    "-t",
-    imageTag,
     ".", // The build context
   ].filter(Boolean) as string[];
 
@@ -814,15 +812,11 @@ USER bun
 WORKDIR /app
 
 ARG TRIGGER_PROJECT_ID
-ARG TRIGGER_DEPLOYMENT_ID
-ARG TRIGGER_DEPLOYMENT_VERSION
 ARG TRIGGER_CONTENT_HASH
 ARG TRIGGER_PROJECT_REF
 ARG NODE_EXTRA_CA_CERTS
 
 ENV TRIGGER_PROJECT_ID=\${TRIGGER_PROJECT_ID} \
-    TRIGGER_DEPLOYMENT_ID=\${TRIGGER_DEPLOYMENT_ID} \
-    TRIGGER_DEPLOYMENT_VERSION=\${TRIGGER_DEPLOYMENT_VERSION} \
     TRIGGER_CONTENT_HASH=\${TRIGGER_CONTENT_HASH} \
     TRIGGER_PROJECT_REF=\${TRIGGER_PROJECT_REF} \
     UV_USE_IO_URING=0 \
@@ -928,15 +922,11 @@ USER node
 WORKDIR /app
 
 ARG TRIGGER_PROJECT_ID
-ARG TRIGGER_DEPLOYMENT_ID
-ARG TRIGGER_DEPLOYMENT_VERSION
 ARG TRIGGER_CONTENT_HASH
 ARG TRIGGER_PROJECT_REF
 ARG NODE_EXTRA_CA_CERTS
 
 ENV TRIGGER_PROJECT_ID=\${TRIGGER_PROJECT_ID} \
-    TRIGGER_DEPLOYMENT_ID=\${TRIGGER_DEPLOYMENT_ID} \
-    TRIGGER_DEPLOYMENT_VERSION=\${TRIGGER_DEPLOYMENT_VERSION} \
     TRIGGER_CONTENT_HASH=\${TRIGGER_CONTENT_HASH} \
     TRIGGER_PROJECT_REF=\${TRIGGER_PROJECT_REF} \
     UV_USE_IO_URING=0 \
@@ -1129,18 +1119,20 @@ function shouldLoad(load?: boolean, push?: boolean) {
 function getOutputOptions({
   imageTag,
   push,
+  load,
   compression,
   compressionLevel,
   forceCompression,
 }: {
   imageTag?: string;
   push?: boolean;
+  load?: boolean;
   compression?: "zstd" | "gzip";
   compressionLevel?: number;
   forceCompression?: boolean;
 }): string[] {
   // Always use OCI media types for compatibility
-  const outputOptions: string[] = ["type=image", "oci-mediatypes=true"];
+  const outputOptions: string[] = ["type=image", "oci-mediatypes=true", "rewrite-timestamp=true"];
 
   if (imageTag) {
     outputOptions.push(`name=${imageTag}`);
@@ -1150,6 +1142,10 @@ function getOutputOptions({
     outputOptions.push("push=true");
   }
 
+  if (load) {
+    outputOptions.push("load=true");
+  }
+
   // Only add compression args when using zstd (gzip is the default, no args needed)
   if (compression === "zstd") {
     outputOptions.push("compression=zstd");
diff --git a/packages/cli-v3/src/entryPoints/managed-index-controller.ts b/packages/cli-v3/src/entryPoints/managed-index-controller.ts
@@ -10,6 +10,7 @@ import { CliApiClient } from "../apiClient.js";
 import { indexWorkerManifest } from "../indexing/indexWorkerManifest.js";
 import { resolveSourceFiles } from "../utilities/sourceFiles.js";
 import { execOptionsForRuntime } from "@trigger.dev/core/v3/build";
+import { writeJSONFile } from "../utilities/fileSystem.js";
 
 async function loadBuildManifest() {
   const manifestContents = await readFile("./build.json", "utf-8");
@@ -88,7 +89,8 @@ async function indexDeployment({
 
     console.log("Writing index.json", process.cwd());
 
-    await writeFile(join(process.cwd(), "index.json"), JSON.stringify(workerManifest, null, 2));
+    const { timings, ...manifestWithoutTimings } = workerManifest;
+    await writeJSONFile(join(process.cwd(), "index.json"), manifestWithoutTimings, true);
 
     const sourceFiles = resolveSourceFiles(buildManifest.sources, workerManifest.tasks);
 
diff --git a/packages/cli-v3/src/entryPoints/managed/env.ts b/packages/cli-v3/src/entryPoints/managed/env.ts
@@ -12,15 +12,15 @@ const DateEnv = z
 const Env = z.object({
   // Set at build time
   TRIGGER_CONTENT_HASH: z.string(),
-  TRIGGER_DEPLOYMENT_ID: z.string(),
-  TRIGGER_DEPLOYMENT_VERSION: z.string(),
   TRIGGER_PROJECT_ID: z.string(),
   TRIGGER_PROJECT_REF: z.string(),
   NODE_ENV: z.string().default("production"),
   NODE_EXTRA_CA_CERTS: z.string().optional(),
   UV_USE_IO_URING: z.string().optional(),
 
   // Set at runtime
+  TRIGGER_DEPLOYMENT_ID: z.string(),
+  TRIGGER_DEPLOYMENT_VERSION: z.string(),
   TRIGGER_WORKLOAD_CONTROLLER_ID: z.string().default(`controller_${randomUUID()}`),
   TRIGGER_ENV_ID: z.string(),
   OTEL_EXPORTER_OTLP_ENDPOINT: z.string().url(),
diff --git a/packages/cli-v3/src/utilities/buildManifest.ts b/packages/cli-v3/src/utilities/buildManifest.ts
@@ -1,10 +1,12 @@
 import { BuildManifest } from "@trigger.dev/core/v3/schemas";
 
 export function buildManifestToJSON(manifest: BuildManifest): BuildManifest {
-  const { deploy, build, ...rest } = manifest;
+  const { deploy, build, externals, ...rest } = manifest;
 
   return {
     ...rest,
+    // sort externals for deterministic builds
+    externals: externals?.slice().sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)),
     deploy: {},
     build: {},
   };
diff --git a/packages/cli-v3/src/utilities/fileSystem.ts b/packages/cli-v3/src/utilities/fileSystem.ts
@@ -1,4 +1,5 @@
 import fsSync from "fs";
+import stringify from "json-stable-stringify";
 import fsModule, { writeFile } from "fs/promises";
 import fs from "node:fs";
 import { homedir, tmpdir } from "node:os";
@@ -159,7 +160,7 @@ export async function safeReadJSONFile(path: string) {
 }
 
 export async function writeJSONFile(path: string, json: any, pretty = false) {
-  await safeWriteFile(path, JSON.stringify(json, undefined, pretty ? 2 : undefined));
+  await safeWriteFile(path, stringify(json, pretty ? { space: 2 } : undefined) ?? "");
 }
 
 // Will create the directory if it doesn't exist
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
