feat: add DST setup

.github/workflows/antithesis_daily.yml

@@ -0,0 +1,81 @@
+name: Antithesis Daily Run
+on:
+  workflow_dispatch:
+    inputs:
+      run_tests:
+        description: "Run Antithesis tests after pushing images"
+        type: boolean
+        default: true
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - "test/antithesis/**"
+      - ".github/workflows/antithesis_daily.yml"
+  schedule:
+    - cron: "0 10 * * *"
+
+jobs:
+  trigger-daily-run:
+    name: Trigger daily run
+    runs-on: shipfox-4vcpu-ubuntu-2404
+    env:
+      ANTITHESIS_PASSWORD: ${{ secrets.ANTITHESIS_PASSWORD }}
+      ANTITHESIS_SLACK_REPORT_RECIPIENT: ${{ secrets.ANTITHESIS_SLACK_REPORT_RECIPIENT }}
+      ANTITHESIS_REPOSITORY: ${{ secrets.ANTITHESIS_REPOSITORY }}
+      OPERATOR_TAG: "v2.10.1"
+      OPERATOR_UTILS_TAG: "v2.0.14"
+      GATEWAY_TAG: "v2.0.24"
+      # LEDGER_PREVIOUS_TAG: "v2.2.47"
+      # temporary: remove this when support for multiple image versions is added
+      LEDGER_PREVIOUS_TAG: "v2.3.0"
+      LEDGER_LATEST_TAG: "v2.3.0"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Just
+        uses: extractions/setup-just@v3
+        with:
+          just-version: "1.40.0"
+
+      - name: Setup Environment
+        uses: ./.github/actions/default
+        with:
+          token: ${{ secrets.NUMARY_GITHUB_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: "NumaryBot"
+          password: ${{ secrets.NUMARY_GITHUB_TOKEN }}
+
+      - name: Login to Antithesis Docker Registry
+        run: |
+          echo '${{ secrets.ANTITHESIS_JSON_KEY }}' | docker login -u _json_key https://us-central1-docker.pkg.dev --password-stdin
+
+      - name: Build and Push Config Image
+        run: |
+          nix develop --impure --command just --justfile ./test/antithesis/Justfile requirements-push
+
+      - name: Run Antithesis Tests
+        uses: antithesishq/[email protected]
+        with:
+          notebook_name: formance-k8s
+          tenant: formance
+          username: ${{ secrets.ANTITHESIS_USERNAME }}
+          password: ${{ secrets.ANTITHESIS_PASSWORD }}
+          github_token: ${{ secrets.NUMARY_GITHUB_TOKEN }}
+          # images: "workload:latest;docker.io/library/postgres:15-alpine;ghcr.io/formancehq/operator:${{ env.OPERATOR_TAG }};ghcr.io/formancehq/operator-utils:${{ env.OPERATOR_UTILS_TAG }};ghcr.io/formancehq/gateway:${{ env.GATEWAY_TAG }};ghcr.io/formancehq/ledger-instrumented:${{ env.LEDGER_PREVIOUS_TAG }};ghcr.io/formancehq/ledger-instrumented:${{ env.LEDGER_LATEST_TAG }}"
+          # temporary: remove this when support for multiple image versions is added
+          images: "workload:latest;docker.io/library/postgres:15-alpine;ghcr.io/formancehq/operator:${{ env.OPERATOR_TAG }};ghcr.io/formancehq/operator-utils:${{ env.OPERATOR_UTILS_TAG }};ghcr.io/formancehq/gateway:${{ env.GATEWAY_TAG }};ghcr.io/formancehq/ledger-instrumented:${{ env.LEDGER_LATEST_TAG }}"
+          config_image: "antithesis-config:daily_run"
+          email_recipients: ${{ secrets.ANTITHESIS_SLACK_REPORT_RECIPIENT }}

.github/workflows/antithesis_release.yml

@@ -0,0 +1,48 @@
+name: Antithesis Instrumented Release
+on:
+  push:
+    tags:
+      - "*"
+
+jobs:
+  push-instrumented-ledger:
+    name: Push instrumented ledger
+    runs-on: shipfox-4vcpu-ubuntu-2404
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Just
+        uses: extractions/setup-just@v3
+        with:
+          just-version: "1.40.0"
+
+      - name: Setup Environment
+        uses: ./.github/actions/default
+        with:
+          token: ${{ secrets.NUMARY_GITHUB_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: "NumaryBot"
+          password: ${{ secrets.NUMARY_GITHUB_TOKEN }}
+
+      - name: Login to Antithesis Docker Registry
+        run: |
+          echo '${{ secrets.ANTITHESIS_JSON_KEY }}' | docker login -u _json_key https://us-central1-docker.pkg.dev --password-stdin
+
+      - name: Build and Push Config Image
+        env:
+          LEDGER_TAG: ${{ env.GITHUB_REF_NAME }}
+        run: |
+          nix develop --impure --command just --justfile ./test/antithesis/image/Justfile push

Earthfile

@@ -50,4 +50,4 @@ deploy:
     RUN kubectl patch Versions.formance.com default -p "{\"spec\":{\"ledger\": \"${tag}\"}}" --type=merge
 
 deploy-staging:
-    BUILD --pass-args core+deploy-staging
+    BUILD --pass-args core+deploy-staging

go.mod

@@ -11,6 +11,7 @@ replace google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215 => google.
 require (
 	github.com/ThreeDotsLabs/watermill v1.4.7
 	github.com/alitto/pond v1.9.2
+	github.com/antithesishq/antithesis-sdk-go v0.5.0
 	github.com/antlr/antlr4/runtime/Go/antlr v1.4.10
 	github.com/bluele/gcache v0.0.2
 	github.com/dop251/goja v0.0.0-20250630131328-58d95d85e994

go.sum

@@ -34,6 +34,10 @@ github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7X
 github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
 github.com/antithesishq/antithesis-sdk-go v0.4.3-default-no-op h1:+OSa/t11TFhqfrX0EOSqQBDJ0YlpmK0rDSiB19dg9M0=
 github.com/antithesishq/antithesis-sdk-go v0.4.3-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
+github.com/antithesishq/antithesis-sdk-go v0.4.3 h1:a2hGdDogClzHzFu20r1z0tzD6zwSWUipiaerAjZVP90=
+github.com/antithesishq/antithesis-sdk-go v0.4.3/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
+github.com/antithesishq/antithesis-sdk-go v0.5.0 h1:cudCFF83pDDANcXFzkQPUHHedfnnIbUO3JMr9fqwFJs=
+github.com/antithesishq/antithesis-sdk-go v0.5.0/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves=
 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY=
 github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=

internal/storage/driver/driver.go

@@ -4,13 +4,15 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"sync"
+	"time"
+
+	"github.com/antithesishq/antithesis-sdk-go/assert"
 	"github.com/formancehq/ledger/internal/storage/common"
 	systemstore "github.com/formancehq/ledger/internal/storage/system"
 	"github.com/formancehq/ledger/internal/tracing"
 	"go.opentelemetry.io/otel/trace"
 	"go.opentelemetry.io/otel/trace/noop"
-	"sync"
-	"time"
 
 	"github.com/alitto/pond"
 	"github.com/formancehq/go-libs/v3/bun/bunpaginate"
@@ -36,9 +38,23 @@ type Driver struct {
 	parallelBucketMigrations int
 }
 
-func (d *Driver) CreateLedger(ctx context.Context, l *ledger.Ledger) (*ledgerstore.Store, error) {
+/*
+CreateLedger creates a new ledger in the system and sets up all necessary database objects.
 
+The function follows these steps:
+ 1. Create a ledger record in the system store (_system.ledgers table)
+ 2. Get the bucket (database schema) for this ledger
+ 3. Check if the bucket is already initialized:
+    a. If initialized: Verify it's up to date and add ledger-specific objects to it
+    b. If not initialized: Create the bucket schema with all necessary tables
+ 4. Return a ledger store that provides an interface to interact with the ledger
+
+Note: This entire process is wrapped in a database transaction, ensuring atomicity.
+If any step fails, the entire transaction is rolled back, preventing partial state.
+*/
+func (d *Driver) CreateLedger(ctx context.Context, l *ledger.Ledger) (*ledgerstore.Store, error) {
 	var ret *ledgerstore.Store
+
 	err := d.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
 		systemStore := d.systemStoreFactory.Create(tx)
 
@@ -54,13 +70,25 @@ func (d *Driver) CreateLedger(ctx context.Context, l *ledger.Ledger) (*ledgersto
 		if err != nil {
 			return fmt.Errorf("checking if bucket is initialized: %w", err)
 		}
+
 		if isInitialized {
 			upToDate, err := b.IsUpToDate(ctx, tx)
 			if err != nil {
 				return fmt.Errorf("checking if bucket is up to date: %w", err)
 			}
 
 			if !upToDate {
+				assert.AlwaysOrUnreachable(
+					// @todo: replace this with a proper flag detailing wether we're
+					// operating a new version of the binary or not.
+					// if we are, we are definitely expecting this to happen.
+					// if we're not, this should be unreachable.
+					false,
+					"Bucket is outdated",
+					map[string]any{
+						"bucket": l.Bucket,
+					},
+				)
 				return ErrBucketOutdated
 			}
 
@@ -77,6 +105,7 @@ func (d *Driver) CreateLedger(ctx context.Context, l *ledger.Ledger) (*ledgersto
 
 		return nil
 	})
+
 	if err != nil {
 		return nil, postgres.ResolveError(err)
 	}

test/antithesis/Justfile

@@ -0,0 +1,27 @@
+run-6min: requirements-push
+    curl --fail \
+        --user "formance:$ANTITHESIS_PASSWORD" \
+        -X POST https://formance.antithesis.com/api/v1/launch_experiment/formance-k8s -d '{ \
+          "params": { \
+            "custom.duration": "0.1", \
+            "antithesis.report.recipients": "'"$ANTITHESIS_SLACK_REPORT_RECIPIENT"'", \
+            "antithesis.config_image": "antithesis-config:daily_run", \
+            "antithesis.images": "workload:latest;docker.io/library/postgres:15-alpine;ghcr.io/formancehq/operator:v2.10.1;ghcr.io/formancehq/operator-utils:v2.0.14;ghcr.io/formancehq/gateway:v2.0.24;ghcr.io/formancehq/ledger-instrumented:'"$LEDGER_LATEST_TAG"'" \
+          } \
+        }'
+
+run-1h: requirements-push
+    curl --fail \
+        --user "formance:$ANTITHESIS_PASSWORD" \
+        -X POST https://formance.antithesis.com/api/v1/launch_experiment/formance-k8s -d '{ \
+          "params": { \
+            "custom.duration": "1", \
+            "antithesis.report.recipients": "'"$ANTITHESIS_SLACK_REPORT_RECIPIENT"'", \
+            "antithesis.config_image": "antithesis-config:daily_run", \
+            "antithesis.images": "workload:latest;docker.io/library/postgres:15-alpine;ghcr.io/formancehq/operator:v2.10.1;ghcr.io/formancehq/operator-utils:v2.0.14;ghcr.io/formancehq/gateway:v2.0.24;ghcr.io/formancehq/ledger-instrumented:"'"$LEDGER_LATEST_TAG"'" \
+          } \
+        }'
+
+requirements-push:
+    just config/push
+    just workload/push

test/antithesis/config/.gitignore

@@ -0,0 +1 @@
+tmp

test/antithesis/config/Dockerfile.config

@@ -0,0 +1,2 @@
+FROM scratch
+COPY tmp/* /manifests/

test/antithesis/config/Justfile

@@ -0,0 +1,18 @@
+build-manifest:
+	rm -f -- tmp/resources.yaml
+	mkdir -p tmp
+	cat manifests/namespace.yaml >> tmp/resources.yaml
+	echo "---" >> tmp/resources.yaml
+	cat manifests/postgres.yaml >> tmp/resources.yaml
+	echo "---" >> tmp/resources.yaml
+	helm template regions oci://ghcr.io/formancehq/helm/regions --version 2.15.2 --namespace formance-systems >> tmp/resources.yaml
+	echo "---" >> tmp/resources.yaml
+	cat manifests/stack.yaml >> tmp/resources.yaml
+	echo "---" >> tmp/resources.yaml
+	yq '.spec.version = strenv(LEDGER_PREVIOUS_TAG)' manifests/ledger.yaml >> tmp/resources.yaml
+	echo "---" >> tmp/resources.yaml
+	cat manifests/workload.yaml >> tmp/resources.yaml
+
+push: build-manifest
+	docker build -f Dockerfile.config -t $ANTITHESIS_REPOSITORY/antithesis-config:daily_run .
+	docker push $ANTITHESIS_REPOSITORY/antithesis-config:daily_run

test/antithesis/config/manifests/ledger.yaml

@@ -0,0 +1,7 @@
+apiVersion: formance.com/v1beta1
+kind: Ledger
+metadata:
+  name: stack0-ledger
+spec:
+  stack: stack0
+  version: LEDGER_VERSION_PLACEHOLDER

test/antithesis/config/manifests/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: formance-systems

test/antithesis/config/manifests/postgres.yaml

@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+  namespace: formance-systems
+spec:
+  selector:
+    app: postgres
+  ports:
+    - protocol: TCP
+      port: 5432
+      targetPort: 5432
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: postgres
+  namespace: formance-systems
+  labels:
+    app: postgres
+spec:
+  containers:
+    - name: postgres
+      image: postgres:15-alpine
+      imagePullPolicy: IfNotPresent
+      args: ["-c", "max_connections=100"]
+      env:
+        - name: "POSTGRES_USER"
+          value: "ledger"
+        - name: "POSTGRES_PASSWORD"
+          value: "ledger"
+        - name: "POSTGRES_DB"
+          value: "ledger"
+        - name: "PGDATA"
+          value: /data/postgres
+      livenessProbe:
+        exec:
+          command: ["pg_isready", "-Uledger"]
+        initialDelaySeconds: 0
+        periodSeconds: 10
+        timeoutSeconds: 5
+        failureThreshold: 5

test/antithesis/config/manifests/stack.yaml

@@ -0,0 +1,33 @@
+apiVersion: formance.com/v1beta1
+kind: Settings
+metadata:
+  name: formance-settings
+spec:
+  key: postgres.*.uri
+  stacks:
+    - "stack0"
+  value: postgresql://ledger:[email protected]:5432?disableSSLMode=true
+---
+apiVersion: formance.com/v1beta1
+kind: Settings
+metadata:
+  name: formance-setting-ledger-image
+spec:
+  key: registries."ghcr.io".images."formancehq/ledger".rewrite
+  stacks:
+    - "stack0"
+  value: "formancehq/ledger-instrumented"
+---
+apiVersion: formance.com/v1beta1
+kind: Stack
+metadata:
+  name: stack0
+spec:
+  versionsFromFile: v2.0
+---
+apiVersion: formance.com/v1beta1
+kind: Gateway
+metadata:
+  name: stack0-gateway
+spec:
+  stack: stack0