test: db migrations

.github/CI.md

@@ -13,18 +13,20 @@ The CI workflow runs on pull request events:
 
 ## Test Run Matrix
 
-| Job                       | Type                                | Runs on PR event (`opened/reopened/synchronize/edited`) | Extra condition                                                                            | What it runs                                                                           |
-| ------------------------- | ----------------------------------- | ------------------------------------------------------- | ------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------- |
-| `test-static`             | Static checks                       | Always                                                  | `needs: build`                                                                             | commitlint title, package checks, typecheck, OpenRPC title check, prettier, eslint     |
-| `test-unit`               | Unit/integration (Nx target `test`) | Always                                                  | `needs: build`                                                                             | `yarn nx affected -t test --base=origin/${{ github.base_ref }} --head=HEAD --parallel` |
-| `build-docs`              | Documentation build                 | Always                                                  | `needs: build`                                                                             | Sphinx docs build for wallet integration guide                                         |
-| `ping-e2e`                | E2E worker (ping app)               | Always                                                  | `needs: build`                                                                             | starts Canton+services and runs Playwright for `@canton-network/example-ping`          |
-| `test-ping-e2e`           | Aggregator/reporting                | Always                                                  | `needs: ping-e2e`, `if: always()`                                                          | fails if `ping-e2e` did not succeed                                                    |
-| `portfolio-e2e`           | E2E worker (portfolio app)          | Always                                                  | `needs: build`                                                                             | starts Canton+services and runs Playwright for `@canton-network/example-portfolio`     |
-| `test-portfolio-e2e`      | Aggregator/reporting                | Always                                                  | `needs: portfolio-e2e`, `if: always()`                                                     | fails if `portfolio-e2e` did not succeed                                               |
-| `wallet-sdk-snippets-e2e` | Wallet SDK snippets E2E             | Always                                                  | `needs: build`                                                                             | snippet tests on matrix `devnet` + `mainnet`                                           |
-| `wallet-sdk-scripts-e2e`  | Wallet SDK scripts E2E              | Always                                                  | `needs: build`                                                                             | example scripts tests on matrix `devnet` + `mainnet`                                   |
-| `wallet-sdk-pkg`          | SDK package validation              | Always                                                  | `needs: build`                                                                             | `yarn script:validate:package`                                                         |
-| `test-wallet-sdk-e2e`     | Aggregator/reporting                | Always                                                  | `needs: [wallet-sdk-snippets-e2e, wallet-sdk-scripts-e2e, wallet-sdk-pkg]`, `if: always()` | fails if any required wallet-sdk e2e/package job did not succeed                       |
+| Job                       | Type                                | Runs on PR event (`opened/reopened/synchronize/edited`) | Extra condition                                                                            | What it runs                                                                                             |
+| ------------------------- | ----------------------------------- | ------------------------------------------------------- | ------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------- |
+| `test-static`             | Static checks                       | Always                                                  | `needs: build`                                                                             | commitlint title, package checks, typecheck, OpenRPC title check, prettier, eslint                       |
+| `test-unit`               | Unit/integration (Nx target `test`) | Always                                                  | `needs: build`                                                                             | `yarn nx affected -t test --base=origin/${{ github.base_ref }} --head=HEAD --parallel`                   |
+| `build-docs`              | Documentation build                 | Always                                                  | `needs: build`                                                                             | Sphinx docs build for wallet integration guide                                                           |
+| `ping-e2e`                | E2E worker (ping app)               | Always                                                  | `needs: build`                                                                             | starts Canton+services and runs Playwright for `@canton-network/example-ping`                            |
+| `test-ping-e2e`           | Aggregator/reporting                | Always                                                  | `needs: ping-e2e`, `if: always()`                                                          | fails if `ping-e2e` did not succeed                                                                      |
+| `portfolio-e2e`           | E2E worker (portfolio app)          | Always                                                  | `needs: build`                                                                             | starts Canton+services and runs Playwright for `@canton-network/example-portfolio`                       |
+| `test-portfolio-e2e`      | Aggregator/reporting                | Always                                                  | `needs: portfolio-e2e`, `if: always()`                                                     | fails if `portfolio-e2e` did not succeed                                                                 |
+| `wallet-sdk-snippets-e2e` | Wallet SDK snippets E2E             | Always                                                  | `needs: build`                                                                             | snippet tests on matrix `devnet` + `mainnet`                                                             |
+| `wallet-sdk-scripts-e2e`  | Wallet SDK scripts E2E              | Always                                                  | `needs: build`                                                                             | example scripts tests on matrix `devnet` + `mainnet`                                                     |
+| `wallet-sdk-pkg`          | SDK package validation              | Always                                                  | `needs: build`                                                                             | `yarn script:validate:package`                                                                           |
+| `test-wallet-sdk-e2e`     | Aggregator/reporting                | Always                                                  | `needs: [wallet-sdk-snippets-e2e, wallet-sdk-scripts-e2e, wallet-sdk-pkg]`, `if: always()` | fails if any required wallet-sdk e2e/package job did not succeed                                         |
+| `test-migrations`         | SQL migration tests                 | Conditional                                             | `needs: build`                                                                             | `yarn nx affected -t test:migrations --base=origin/${{ github.base_ref }} --head=HEAD --parallel=1`      |
+| `check-migration-lock`    | Test SQL migration immutability     | Conditional                                             | None // TODO verify if doesnt need build                                                   | `yarn nx affected -t migrations:check-lock  --base=origin/${{ github.base_ref }} --head=HEAD --parallel` |
 
 The workflow uses aggregator wrappers (`test-ping-e2e`, `test-portfolio-e2e`, and `test-wallet-sdk-e2e`) that always run and validate the success of the corresponding worker jobs.

.github/workflows/build.yml

@@ -226,14 +226,56 @@ jobs:
                     --head=HEAD \
                     --parallel
 
+    check-migration-lock:
+        # Verifies migrations.lock.json is in sync with src/migrations for any
+        # affected SQL store package that defines a `migrations:check-lock`
+        # script. Catches accidental migration edits
+        needs: build
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v6
+              with:
+                  fetch-depth: 0
+
+            - uses: ./.github/actions/setup_yarn
+
+            - name: Check migration lock files
+              run: |
+                  yarn nx affected -t migrations:check-lock \
+                    --base=origin/${{ github.base_ref }} \
+                    --head=HEAD \
+                    --parallel
+
+    test-migrations:
+        # Runs SQL migration tests for any affected package that defines a
+        # `test:migrations` script
+        runs-on: ubuntu-latest
+        needs: build
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v6
+              with:
+                  fetch-depth: 0
+
+            - uses: ./.github/actions/setup_yarn
+
+            - name: Run affected migration tests
+              run: |
+                  yarn nx affected -t test:migrations \
+                    --base=origin/${{ github.base_ref }} \
+                    --head=HEAD \
+                    --parallel=1
+
     ping-e2e:
-        name: ping-e2e (${{ matrix.network }})
+        name: ping-e2e (${{ matrix.network }}, ${{ matrix.db }})
         runs-on: ubuntu-latest
         needs: [version-config, hydrate-canton-caches]
         strategy:
             fail-fast: false
             matrix:
                 network: [devnet, mainnet]
+                db: [sqlite, postgres]
         steps:
             - name: Checkout
               uses: actions/checkout@v6
@@ -247,6 +289,45 @@ jobs:
                   network: ${{ matrix.network }}
                   canton_version: ${{ matrix.network == 'devnet' && needs.version-config.outputs.devnet_canton_version || needs.version-config.outputs.mainnet_canton_version }}
 
+            - name: Start Postgres container
+              if: matrix.db == 'postgres'
+              run: |
+                  docker run -d --name wk-postgres \
+                    -e POSTGRES_USER=postgres \
+                    -e POSTGRES_PASSWORD=postgres \
+                    -e POSTGRES_DB=postgres \
+                    -p 5432:5432 \
+                    --health-cmd="pg_isready -U postgres" \
+                    --health-interval=2s \
+                    postgres:16-alpine
+
+                  until [ "$(docker inspect -f '{{.State.Health.Status}}' wk-postgres)" = "healthy" ]; do
+                    sleep 5
+                  done
+                  docker inspect -f '{{.State.Health.Status}}' wk-postgres | grep -q healthy
+
+            - name: Point gateway config at Postgres
+              if: matrix.db == 'postgres'
+              run: |
+                  CONFIG=wallet-gateway/test/config.json
+                  jq '.store.connection = {
+                        type: "postgres",
+                        host: "localhost",
+                        port: 5432,
+                        user: "postgres",
+                        password: "postgres",
+                        database: "wallet"
+                      }
+                      | .signingStore.connection = {
+                        type: "postgres",
+                        host: "localhost",
+                        port: 5432,
+                        user: "postgres",
+                        password: "postgres",
+                        database: "signing"
+                      }' "$CONFIG" > "$CONFIG.tmp"
+                  mv "$CONFIG.tmp" "$CONFIG"
+
             - name: Start remote WK
               run: |
                   BLOCKDAEMON_API_URL="${{ vars.BLOCKDAEMON_API_URL }}" \
@@ -268,17 +349,21 @@ jobs:
             - uses: actions/upload-artifact@v7
               if: ${{ !cancelled() }}
               with:
-                  name: example-ping-playwright-report-${{ matrix.network }}
+                  name: example-ping-playwright-report-${{ matrix.network }}-${{ matrix.db }}
                   path: examples/ping/playwright-report/
                   retention-days: 5
 
             - uses: actions/upload-artifact@v7
               if: ${{ !cancelled() }}
               with:
-                  name: ping-wallet-gateway-remote-log-${{ matrix.network }}
+                  name: ping-wallet-gateway-remote-log-${{ matrix.network }}-${{ matrix.db }}
                   path: wallet-gateway-remote.log
                   retention-days: 5
 
+            - name: Stop Postgres container
+              if: ${{ always() && matrix.db == 'postgres' }}
+              run: docker rm -f wk-postgres || true
+
     test-ping-e2e:
         runs-on: ubuntu-latest
         needs: ping-e2e

core/signing-store-sql/migrations.lock.json

@@ -0,0 +1,8 @@
+{
+    "version": 1,
+    "migrations": {
+        "001-init": "ce148408f89c3d2589835991db5c61cf0fc927e362278bcf1243c6d7db53b397",
+        "002-add-signed-at": "3eba642ef9b54406db9868e40da85708b793459e21479614829158d02b3608ab",
+        "003-alter-date-fields": "c5195429926500de1fb010fd33ab4b5f63ecd35b27d4f4cb62cf3b426eef9727"
+    }
+}

core/signing-store-sql/package.json

@@ -23,7 +23,10 @@
         "flatpack": "yarn pack --out \"$FLATPACK_OUTDIR\"",
         "clean": "tsc -b --clean; rm -rf dist",
         "test": "vitest run --project node --passWithNoTests",
-        "test:coverage": "vitest run --project node --coverage --passWithNoTests"
+        "test:coverage": "vitest run --project node --coverage --passWithNoTests",
+        "test:migrations": "vitest run --project migrations",
+        "migrations:check-lock": "tsx ../../scripts/src/check-migration-lock.ts",
+        "migrations:update-lock": "tsx ../../scripts/src/check-migration-lock.ts --update"
     },
     "dependencies": {
         "@canton-network/core-ledger-client": "workspace:^",
@@ -41,6 +44,7 @@
     },
     "devDependencies": {
         "@swc/core": "^1.15.18",
+        "@testcontainers/postgresql": "^11.14.0",
         "@types/better-sqlite3": "^7.6.13",
         "@types/pg": "^8.18.0",
         "@vitest/coverage-v8": "^4.1.2",

core/signing-store-sql/src/migrations-test/README.md

@@ -0,0 +1,25 @@
+# Migration tests for wallet-store-sql
+
+Runs migrations and tests their effects against all supported SQL dialects
+
+`/migration-test/data/`
+vitest test files that must reflect `/migrations/` filenames with `.test.ts` extension
+
+`/migration-test/seeds/`
+utils for inserting rows satisfying table schema that reflects that state of migrations being applied up to the migration with related filename
+
+`/migration-test/helpers.ts`
+utils for validating schema
+
+`/migration-test/coverage.test.ts`
+test that validates that every migration has a test. if not - fail the tests. intention is to assure that added migrations are validated against all supported SQL dialects
+
+`/migration-test/global-setup.ts`
+spins up postgres with testcontainers
+
+`/migrations.lock.json`
+contains hashes of all migration files. intention is to prevent accidental modifications of existing migrations.
+
+`yarn migrations:check-lock` recomputes hashes from migration files and compares them against lock file
+
+`yarn migrations:update-lock` recomputes hashes and updates lock file. use it after adding a new migration, or in case of intentionally editing old migration (i.e. when adding a comment), altering logic in old migrations should be avoided

core/signing-store-sql/src/migrations-test/coverage.test.ts

@@ -0,0 +1,46 @@
+// Copyright (c) 2025-2026 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { readdirSync } from 'node:fs'
+import { fileURLToPath } from 'node:url'
+import { dirname, resolve } from 'node:path'
+import { describe, expect, test } from 'vitest'
+
+const here = dirname(fileURLToPath(import.meta.url))
+const MIGRATIONS_DIR = resolve(here, '../migrations')
+const TESTS_DIR = resolve(here, 'data')
+
+const migrationNames = (): string[] =>
+    readdirSync(MIGRATIONS_DIR)
+        .filter((f) => f.endsWith('.ts'))
+        .map((f) => f.replace(/\.ts$/, ''))
+
+const testNames = (): string[] =>
+    readdirSync(TESTS_DIR)
+        .filter((f) => f.endsWith('.test.ts'))
+        .map((f) => f.replace(/\.test\.ts$/, ''))
+
+describe('migration test coverage', () => {
+    const migrations = new Set(migrationNames())
+    const tests = new Set(testNames())
+
+    test('every migration has a sibling test file with the same name', () => {
+        const missing = [...migrations]
+            .filter((name) => !tests.has(name))
+            .map((name) => `expected src/migrations-test/data/${name}.test.ts`)
+        expect(
+            missing,
+            `Migrations without corresponding tests:\n  ${missing.join('\n  ')}`
+        ).toEqual([])
+    })
+
+    test('every migration test file matches an existing migration', () => {
+        const orphans = [...tests]
+            .filter((name) => !migrations.has(name))
+            .map((name) => `no migration src/migrations/${name}.ts found`)
+        expect(
+            orphans,
+            `Test files without a backing migration:\n  ${orphans.join('\n  ')}`
+        ).toEqual([])
+    })
+})