From ecdc30ec09c7ca19bb993dc31f31f2661c5024d2 Mon Sep 17 00:00:00 2001 From: vindard <17693119+vindard@users.noreply.github.com> Date: Tue, 26 May 2026 21:34:39 -0400 Subject: [PATCH 1/3] chore(deps): bump es-entity to 0.10.37 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit es-entity 0.10.37 contains the direction-aware NULL fallback in cursor WHERE clauses for nullable sort columns (GaloyMoney/es-entity#137) and the opt-in `nullable` column attribute for non-Option Rust types (GaloyMoney/es-entity#138). The macro change in #137 alters the DESC SQL emitted for `Option<...>` columns — for cala-ledger this affects `Account.external_id` and `AccountSet.external_id` pagination. New fingerprints replace the old ones in `cala-ledger/.sqlx/`; semantics are unchanged for non-NULL rows and now also include NULL rows on page 2+ for DESC (previously dropped). Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- ...fd472474dcea39c895bf0d6815c035c397abf1fc5b44c069.json} | 7 ++++--- ...26af552bfc109d715d477118083ddabf96aa88ae1cb65c7c.json} | 7 ++++--- ...6989b156826656095cdae392dd9e347a9c2144784ec44c7d.json} | 4 ++-- ...d7d053e38a14352e4b3f590c9124173f4c08f95428ccc21b.json} | 4 ++-- ...5a44a631dcc321f785e3f8b549a89cef5f5fd7cd17cdb5cd.json} | 4 ++-- 7 files changed, 19 insertions(+), 17 deletions(-) rename cala-ledger/.sqlx/{query-8dfaa15816547fecb125c30444481590b22abd0ed2199ecf0a099991850db17a.json => query-03994377e116e70dfd472474dcea39c895bf0d6815c035c397abf1fc5b44c069.json} (79%) rename cala-ledger/.sqlx/{query-446db9f053f56473bb1bfc0a86b626c0bb2b31d058f247151439a3f656b9de4e.json => query-60c8da85d099dfdd26af552bfc109d715d477118083ddabf96aa88ae1cb65c7c.json} (94%) rename cala-ledger/.sqlx/{query-af56d9b7a5a837d6427edb0e25b40de901697ff77c90fabd863bb54acf5ec5b9.json => query-e750e171f2397f146989b156826656095cdae392dd9e347a9c2144784ec44c7d.json} (63%) rename cala-ledger/.sqlx/{query-1d1a35bc0d3a02c26f776ab068e6185018b536cb9da4876057d2762ff6dc0605.json => query-f3883e5f7bd52bf7d7d053e38a14352e4b3f590c9124173f4c08f95428ccc21b.json} (66%) rename cala-ledger/.sqlx/{query-6a6577f96ae0f098a4539fc36d267fcf8fa836ce7c6f54c42116256b6fc22735.json => query-fe49f46b26ce060e5a44a631dcc321f785e3f8b549a89cef5f5fd7cd17cdb5cd.json} (63%) diff --git a/Cargo.lock b/Cargo.lock index 7d0ff4bf4..51bcd7723 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -847,9 +847,9 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "es-entity" -version = "0.10.36" +version = "0.10.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bffb4c045095c29c481e97358b072497865b0c45ff5ff23fdf57eaa069e59971" +checksum = "ddf124151df32af341cd82f3ca6e014c50ed948eae99f5380f84efeabf42e1a9" dependencies = [ "chrono", "derive_builder", @@ -872,9 +872,9 @@ dependencies = [ [[package]] name = "es-entity-macros" -version = "0.10.36" +version = "0.10.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c1bb41f0f8f49452fa1ffd0377ddb180df35a8fcee7024a3d2a2974cd6c2d7e" +checksum = "cd38eb6d28a6e760303bc8b3940d9f114f9c49ec0bc01a02b226a73ea06d72a6" dependencies = [ "convert_case", "darling 0.23.0", diff --git a/Cargo.toml b/Cargo.toml index e0b715846..f353ebeda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ cala-types = { path = "cala-ledger-core-types", package = "cala-ledger-core-type cala-tracing = { path = "cala-tracing", version = "0.15.10-dev" } cala-ledger = { path = "cala-ledger", version = "0.15.10-dev" } -es-entity = "0.10.36" +es-entity = "0.10.37" job = { version = "0.6.25", features = ["es-entity"] } obix = { version = "0.2.27", default-features = false } diff --git a/cala-ledger/.sqlx/query-8dfaa15816547fecb125c30444481590b22abd0ed2199ecf0a099991850db17a.json b/cala-ledger/.sqlx/query-03994377e116e70dfd472474dcea39c895bf0d6815c035c397abf1fc5b44c069.json similarity index 79% rename from cala-ledger/.sqlx/query-8dfaa15816547fecb125c30444481590b22abd0ed2199ecf0a099991850db17a.json rename to cala-ledger/.sqlx/query-03994377e116e70dfd472474dcea39c895bf0d6815c035c397abf1fc5b44c069.json index 5a40d2f5b..79f019fc8 100644 --- a/cala-ledger/.sqlx/query-8dfaa15816547fecb125c30444481590b22abd0ed2199ecf0a099991850db17a.json +++ b/cala-ledger/.sqlx/query-03994377e116e70dfd472474dcea39c895bf0d6815c035c397abf1fc5b44c069.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE job_executions\n SET state = 'pending', execute_at = $1, attempt_index = attempt_index + 1, poller_instance_id = NULL\n WHERE state = 'running' AND alive_at < $1::timestamptz\n AND job_type = ANY($2)\n AND poller_instance_id IS DISTINCT FROM $3\n RETURNING id as id\n ", + "query": "\n UPDATE job_executions\n SET state = 'pending', execute_at = $4, attempt_index = attempt_index + 1, poller_instance_id = NULL\n WHERE state = 'running' AND alive_at < $1::timestamptz\n AND job_type = ANY($2)\n AND poller_instance_id IS DISTINCT FROM $3\n RETURNING id as id\n ", "describe": { "columns": [ { @@ -13,12 +13,13 @@ "Left": [ "Timestamptz", "TextArray", - "Uuid" + "Uuid", + "Timestamptz" ] }, "nullable": [ false ] }, - "hash": "8dfaa15816547fecb125c30444481590b22abd0ed2199ecf0a099991850db17a" + "hash": "03994377e116e70dfd472474dcea39c895bf0d6815c035c397abf1fc5b44c069" } diff --git a/cala-ledger/.sqlx/query-446db9f053f56473bb1bfc0a86b626c0bb2b31d058f247151439a3f656b9de4e.json b/cala-ledger/.sqlx/query-60c8da85d099dfdd26af552bfc109d715d477118083ddabf96aa88ae1cb65c7c.json similarity index 94% rename from cala-ledger/.sqlx/query-446db9f053f56473bb1bfc0a86b626c0bb2b31d058f247151439a3f656b9de4e.json rename to cala-ledger/.sqlx/query-60c8da85d099dfdd26af552bfc109d715d477118083ddabf96aa88ae1cb65c7c.json index 55b83c2d1..82b5c2a21 100644 --- a/cala-ledger/.sqlx/query-446db9f053f56473bb1bfc0a86b626c0bb2b31d058f247151439a3f656b9de4e.json +++ b/cala-ledger/.sqlx/query-60c8da85d099dfdd26af552bfc109d715d477118083ddabf96aa88ae1cb65c7c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n WITH eligible AS (\n SELECT id, queue_id, execute_at, execution_state_json, attempt_index\n FROM job_executions\n WHERE state = 'pending'\n AND job_type = ANY($4)\n AND NOT EXISTS (\n SELECT 1 FROM job_executions AS running\n WHERE running.state = 'running'\n AND running.queue_id IS NOT NULL\n AND running.queue_id = job_executions.queue_id\n )\n ),\n min_wait AS (\n SELECT MIN(execute_at) - $2::timestamptz AS wait_time\n FROM eligible\n WHERE execute_at > $2::timestamptz\n ),\n candidates AS (\n SELECT id, execution_state_json AS data_json, attempt_index,\n ROW_NUMBER() OVER (\n PARTITION BY COALESCE(queue_id, id::text)\n ORDER BY execute_at\n ) AS rn\n FROM eligible\n WHERE execute_at <= $2::timestamptz\n ),\n selected_jobs AS (\n SELECT je.id, c.data_json, c.attempt_index\n FROM candidates c\n JOIN job_executions je ON je.id = c.id\n WHERE c.rn = 1\n ORDER BY je.execute_at ASC\n LIMIT $1\n FOR UPDATE OF je\n ),\n updated AS (\n UPDATE job_executions AS je\n SET state = 'running', alive_at = $2, execute_at = NULL, poller_instance_id = $3\n FROM selected_jobs\n WHERE je.id = selected_jobs.id\n AND je.state = 'pending'\n RETURNING je.id, selected_jobs.data_json, je.attempt_index\n )\n SELECT * FROM (\n SELECT\n u.id AS \"id?: JobId\",\n u.data_json AS \"data_json?: JsonValue\",\n u.attempt_index AS \"attempt_index?\",\n NULL::INTERVAL AS \"max_wait?: PgInterval\"\n FROM updated u\n UNION ALL\n SELECT\n NULL::UUID AS \"id?: JobId\",\n NULL::JSONB AS \"data_json?: JsonValue\",\n NULL::INT AS \"attempt_index?\",\n mw.wait_time AS \"max_wait?: PgInterval\"\n FROM min_wait mw\n WHERE NOT EXISTS (SELECT 1 FROM updated)\n ) AS result\n ", + "query": "\n WITH eligible AS (\n SELECT id, queue_id, execute_at, execution_state_json, attempt_index\n FROM job_executions\n WHERE state = 'pending'\n AND job_type = ANY($4)\n AND NOT EXISTS (\n SELECT 1 FROM job_executions AS running\n WHERE running.state = 'running'\n AND running.queue_id IS NOT NULL\n AND running.queue_id = job_executions.queue_id\n )\n ),\n min_wait AS (\n SELECT MIN(execute_at) - $2::timestamptz AS wait_time\n FROM eligible\n WHERE execute_at > $2::timestamptz\n ),\n candidates AS (\n SELECT id, execution_state_json AS data_json, attempt_index,\n ROW_NUMBER() OVER (\n PARTITION BY COALESCE(queue_id, id::text)\n ORDER BY execute_at\n ) AS rn\n FROM eligible\n WHERE execute_at <= $2::timestamptz\n ),\n selected_jobs AS (\n SELECT je.id, c.data_json, c.attempt_index\n FROM candidates c\n JOIN job_executions je ON je.id = c.id\n WHERE c.rn = 1\n ORDER BY je.execute_at ASC\n LIMIT $1\n FOR UPDATE OF je\n ),\n updated AS (\n UPDATE job_executions AS je\n SET state = 'running', alive_at = $5, execute_at = NULL, poller_instance_id = $3\n FROM selected_jobs\n WHERE je.id = selected_jobs.id\n AND je.state = 'pending'\n RETURNING je.id, selected_jobs.data_json, je.attempt_index\n )\n SELECT * FROM (\n SELECT\n u.id AS \"id?: JobId\",\n u.data_json AS \"data_json?: JsonValue\",\n u.attempt_index AS \"attempt_index?\",\n NULL::INTERVAL AS \"max_wait?: PgInterval\"\n FROM updated u\n UNION ALL\n SELECT\n NULL::UUID AS \"id?: JobId\",\n NULL::JSONB AS \"data_json?: JsonValue\",\n NULL::INT AS \"attempt_index?\",\n mw.wait_time AS \"max_wait?: PgInterval\"\n FROM min_wait mw\n WHERE NOT EXISTS (SELECT 1 FROM updated)\n ) AS result\n ", "describe": { "columns": [ { @@ -29,7 +29,8 @@ "Int8", "Timestamptz", "Uuid", - "TextArray" + "TextArray", + "Timestamptz" ] }, "nullable": [ @@ -39,5 +40,5 @@ null ] }, - "hash": "446db9f053f56473bb1bfc0a86b626c0bb2b31d058f247151439a3f656b9de4e" + "hash": "60c8da85d099dfdd26af552bfc109d715d477118083ddabf96aa88ae1cb65c7c" } diff --git a/cala-ledger/.sqlx/query-af56d9b7a5a837d6427edb0e25b40de901697ff77c90fabd863bb54acf5ec5b9.json b/cala-ledger/.sqlx/query-e750e171f2397f146989b156826656095cdae392dd9e347a9c2144784ec44c7d.json similarity index 63% rename from cala-ledger/.sqlx/query-af56d9b7a5a837d6427edb0e25b40de901697ff77c90fabd863bb54acf5ec5b9.json rename to cala-ledger/.sqlx/query-e750e171f2397f146989b156826656095cdae392dd9e347a9c2144784ec44c7d.json index 404e3483d..b3483e099 100644 --- a/cala-ledger/.sqlx/query-af56d9b7a5a837d6427edb0e25b40de901697ff77c90fabd863bb54acf5ec5b9.json +++ b/cala-ledger/.sqlx/query-e750e171f2397f146989b156826656095cdae392dd9e347a9c2144784ec44c7d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH entities AS (SELECT external_id, id FROM cala_account_sets WHERE ((external_id IS NOT DISTINCT FROM $3) AND COALESCE(id < $2, true) OR COALESCE(external_id < $3, external_id IS NOT NULL)) ORDER BY external_id DESC NULLS LAST, id DESC LIMIT $1) SELECT i.id AS \"entity_id: Repo__Id\", e.sequence, e.event, CASE WHEN $4 THEN e.context ELSE NULL::jsonb END as \"context: es_entity::ContextData\", e.recorded_at FROM entities i JOIN cala_account_set_events e ON i.id = e.id ORDER BY i.external_id desc nulls last, i.id desc, i.id, e.sequence", + "query": "WITH entities AS (SELECT external_id, id FROM cala_account_sets WHERE ((external_id IS NOT DISTINCT FROM $3) AND COALESCE(id < $2, true) OR COALESCE(external_id < $3, $2 IS NULL OR (external_id IS NULL AND $3 IS NOT NULL))) ORDER BY external_id DESC NULLS LAST, id DESC LIMIT $1) SELECT i.id AS \"entity_id: Repo__Id\", e.sequence, e.event, CASE WHEN $4 THEN e.context ELSE NULL::jsonb END as \"context: es_entity::ContextData\", e.recorded_at FROM entities i JOIN cala_account_set_events e ON i.id = e.id ORDER BY i.external_id desc nulls last, i.id desc, i.id, e.sequence", "describe": { "columns": [ { @@ -45,5 +45,5 @@ false ] }, - "hash": "af56d9b7a5a837d6427edb0e25b40de901697ff77c90fabd863bb54acf5ec5b9" + "hash": "e750e171f2397f146989b156826656095cdae392dd9e347a9c2144784ec44c7d" } diff --git a/cala-ledger/.sqlx/query-1d1a35bc0d3a02c26f776ab068e6185018b536cb9da4876057d2762ff6dc0605.json b/cala-ledger/.sqlx/query-f3883e5f7bd52bf7d7d053e38a14352e4b3f590c9124173f4c08f95428ccc21b.json similarity index 66% rename from cala-ledger/.sqlx/query-1d1a35bc0d3a02c26f776ab068e6185018b536cb9da4876057d2762ff6dc0605.json rename to cala-ledger/.sqlx/query-f3883e5f7bd52bf7d7d053e38a14352e4b3f590c9124173f4c08f95428ccc21b.json index 5fe4e5a6b..4c3244b9f 100644 --- a/cala-ledger/.sqlx/query-1d1a35bc0d3a02c26f776ab068e6185018b536cb9da4876057d2762ff6dc0605.json +++ b/cala-ledger/.sqlx/query-f3883e5f7bd52bf7d7d053e38a14352e4b3f590c9124173f4c08f95428ccc21b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH entities AS (SELECT external_id, id FROM cala_account_sets WHERE COALESCE(name = $1, $1 IS NULL) AND ((external_id IS NOT DISTINCT FROM $4) AND COALESCE(id < $3, true) OR COALESCE(external_id < $4, external_id IS NOT NULL)) ORDER BY external_id DESC NULLS LAST, id DESC LIMIT $2) SELECT i.id AS \"entity_id: Repo__Id\", e.sequence, e.event, CASE WHEN $5 THEN e.context ELSE NULL::jsonb END as \"context: es_entity::ContextData\", e.recorded_at FROM entities i JOIN cala_account_set_events e ON i.id = e.id ORDER BY i.external_id desc nulls last, i.id desc, i.id, e.sequence", + "query": "WITH entities AS (SELECT external_id, id FROM cala_account_sets WHERE COALESCE(name = $1, $1 IS NULL) AND ((external_id IS NOT DISTINCT FROM $4) AND COALESCE(id < $3, true) OR COALESCE(external_id < $4, $3 IS NULL OR (external_id IS NULL AND $4 IS NOT NULL))) ORDER BY external_id DESC NULLS LAST, id DESC LIMIT $2) SELECT i.id AS \"entity_id: Repo__Id\", e.sequence, e.event, CASE WHEN $5 THEN e.context ELSE NULL::jsonb END as \"context: es_entity::ContextData\", e.recorded_at FROM entities i JOIN cala_account_set_events e ON i.id = e.id ORDER BY i.external_id desc nulls last, i.id desc, i.id, e.sequence", "describe": { "columns": [ { @@ -46,5 +46,5 @@ false ] }, - "hash": "1d1a35bc0d3a02c26f776ab068e6185018b536cb9da4876057d2762ff6dc0605" + "hash": "f3883e5f7bd52bf7d7d053e38a14352e4b3f590c9124173f4c08f95428ccc21b" } diff --git a/cala-ledger/.sqlx/query-6a6577f96ae0f098a4539fc36d267fcf8fa836ce7c6f54c42116256b6fc22735.json b/cala-ledger/.sqlx/query-fe49f46b26ce060e5a44a631dcc321f785e3f8b549a89cef5f5fd7cd17cdb5cd.json similarity index 63% rename from cala-ledger/.sqlx/query-6a6577f96ae0f098a4539fc36d267fcf8fa836ce7c6f54c42116256b6fc22735.json rename to cala-ledger/.sqlx/query-fe49f46b26ce060e5a44a631dcc321f785e3f8b549a89cef5f5fd7cd17cdb5cd.json index 96b22c976..850c5e50c 100644 --- a/cala-ledger/.sqlx/query-6a6577f96ae0f098a4539fc36d267fcf8fa836ce7c6f54c42116256b6fc22735.json +++ b/cala-ledger/.sqlx/query-fe49f46b26ce060e5a44a631dcc321f785e3f8b549a89cef5f5fd7cd17cdb5cd.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH entities AS (SELECT external_id, id FROM cala_accounts WHERE ((external_id IS NOT DISTINCT FROM $3) AND COALESCE(id < $2, true) OR COALESCE(external_id < $3, external_id IS NOT NULL)) ORDER BY external_id DESC NULLS LAST, id DESC LIMIT $1) SELECT i.id AS \"entity_id: Repo__Id\", e.sequence, e.event, CASE WHEN $4 THEN e.context ELSE NULL::jsonb END as \"context: es_entity::ContextData\", e.recorded_at FROM entities i JOIN cala_account_events e ON i.id = e.id ORDER BY i.external_id desc nulls last, i.id desc, i.id, e.sequence", + "query": "WITH entities AS (SELECT external_id, id FROM cala_accounts WHERE ((external_id IS NOT DISTINCT FROM $3) AND COALESCE(id < $2, true) OR COALESCE(external_id < $3, $2 IS NULL OR (external_id IS NULL AND $3 IS NOT NULL))) ORDER BY external_id DESC NULLS LAST, id DESC LIMIT $1) SELECT i.id AS \"entity_id: Repo__Id\", e.sequence, e.event, CASE WHEN $4 THEN e.context ELSE NULL::jsonb END as \"context: es_entity::ContextData\", e.recorded_at FROM entities i JOIN cala_account_events e ON i.id = e.id ORDER BY i.external_id desc nulls last, i.id desc, i.id, e.sequence", "describe": { "columns": [ { @@ -45,5 +45,5 @@ false ] }, - "hash": "6a6577f96ae0f098a4539fc36d267fcf8fa836ce7c6f54c42116256b6fc22735" + "hash": "fe49f46b26ce060e5a44a631dcc321f785e3f8b549a89cef5f5fd7cd17cdb5cd" } From 54cf3c0c0573e8b01530c16cf92a05e7ae524fa0 Mon Sep 17 00:00:00 2001 From: vindard <17693119+vindard@users.noreply.github.com> Date: Tue, 26 May 2026 23:21:05 -0400 Subject: [PATCH 2/3] chore: replace docker-compose with nix process-compose for dev deps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align with lana-bank's docker-free dev tooling: contributors no longer need Docker (or podman) installed. start-deps now runs Postgres (with pg_stat_statements preloaded) plus the OTel collector directly via process-compose-flake, with data stored under .nix-deps/. Drops docker-compose.yml, dev/bin/{clean-deps,docker-compose-up}.sh, and nix/podman-runner.nix. The otel-agent config switches the deprecated `logging` exporter to `debug` for compatibility with otelcol-contrib 0.144+ (nixpkgs default). CI surfaces (.github workflows + Concourse pipeline) are unchanged — both already invoke nix run .#nextest / .#perf, whose internals now use the process-compose path. Refs GaloyMoney/volcano-wip#772. --- Makefile | 32 ++--- README.md | 4 - dev/bin/clean-deps.sh | 17 --- dev/bin/docker-compose-up.sh | 28 ----- dev/otel-agent-config.yaml | 6 +- docker-compose.yml | 30 ----- flake.lock | 16 +++ flake.nix | 229 +++++++++++++++++++++++++++-------- nix/podman-runner.nix | 123 ------------------- 9 files changed, 216 insertions(+), 269 deletions(-) delete mode 100755 dev/bin/clean-deps.sh delete mode 100755 dev/bin/docker-compose-up.sh delete mode 100644 docker-compose.yml delete mode 100644 nix/podman-runner.nix diff --git a/Makefile b/Makefile index c8d737590..407ed494a 100644 --- a/Makefile +++ b/Makefile @@ -1,24 +1,26 @@ +NIX_DEPS_DIR := .nix-deps + +.PHONY: start-deps clean-deps setup-db reset-deps reset-deps-perf sqlx-prepare check-code build test-in-ci event-schemas check-event-schemas next-watch rust-example + next-watch: cargo watch -s 'cargo nextest run' -clean-deps: - ./dev/bin/clean-deps.sh - start-deps: - ./dev/bin/docker-compose-up.sh integration-deps + @mkdir -p $(NIX_DEPS_DIR) + nix run .#nix-deps-base -- up -D + nix run .#nix-deps-base -- project is-ready --wait + +clean-deps: + -nix run .#nix-deps-base -- down + chmod -R u+w $(NIX_DEPS_DIR) 2>/dev/null || true + rm -rf $(NIX_DEPS_DIR) setup-db: - @echo "Waiting for PostgreSQL and running migrations..." - @for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30; do \ - (cd cala-ledger && cargo sqlx migrate run 2>/dev/null) && echo "Migrations complete" && exit 0; \ - echo "Attempt $$i: Database not ready, waiting..."; \ - sleep 1; \ - done; \ - echo "Database failed to become ready after 30 attempts"; \ - cd cala-ledger && cargo sqlx migrate run - -reset-deps: clean-deps start-deps setup-db -reset-deps-perf: clean-deps start-deps setup-db + nix run .#setup-db-dev + +reset-deps: clean-deps start-deps + +reset-deps-perf: clean-deps start-deps psql postgres://user:password@localhost:5432/pg -f ./cala-perf/pg-tools/setup.sql rust-example: diff --git a/README.md b/README.md index 8b10933f6..f2b2a897f 100644 --- a/README.md +++ b/README.md @@ -38,10 +38,6 @@ Cala is a robust ledger system developed by Galoy, designed to handle complex fi source ~/.bashrc ``` -#### Docker - -- Choose the install method for your system https://docs.docker.com/desktop/ - ### Testing Run unit tests with: diff --git a/dev/bin/clean-deps.sh b/dev/bin/clean-deps.sh deleted file mode 100755 index 83fc2433d..000000000 --- a/dev/bin/clean-deps.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# ── Pick container engine ─────────────────────────────────────────────────────── -if [[ -n "${ENGINE_DEFAULT:-}" ]]; then - ENGINE="$ENGINE_DEFAULT" -else - ENGINE=docker -fi - -if ! command -v "$ENGINE" >/dev/null 2>&1; then - printf 'Error: requested engine "%s" not found in $PATH\n' "$ENGINE" >&2 - exit 1 -fi - -# ── Down ──────────────────────────────────────────────────────────────────────── -exec "$ENGINE" compose -f docker-compose.yml down -v -t 2 diff --git a/dev/bin/docker-compose-up.sh b/dev/bin/docker-compose-up.sh deleted file mode 100755 index 94fe9016a..000000000 --- a/dev/bin/docker-compose-up.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# ── Pick container engine ─────────────────────────────────────────────────────── -if [[ -n "${ENGINE_DEFAULT:-}" ]]; then # honour explicit choice - ENGINE="$ENGINE_DEFAULT" -else # otherwise prefer docker - ENGINE=docker -fi - -# ensure the binary is on PATH -if ! command -v "$ENGINE" >/dev/null 2>&1; then - printf 'Error: requested engine "%s" not found in $PATH\n' "$ENGINE" >&2 - exit 1 -fi - -# ── Pull images first (prevents concurrent map writes) ───────────────────────── -# Only pull in CI to avoid slow re-pulls during local development -if [[ "${CI:-false}" == "true" ]]; then - echo "Pulling Docker images..." - "$ENGINE" compose -f docker-compose.yml pull -fi - -# ── Up ────────────────────────────────────────────────────────────────────────── -echo "Starting services..." -"$ENGINE" compose -f docker-compose.yml up -d "$@" - -wait4x postgresql ${PG_CON} --timeout 120s diff --git a/dev/otel-agent-config.yaml b/dev/otel-agent-config.yaml index c5ae82bec..247628585 100644 --- a/dev/otel-agent-config.yaml +++ b/dev/otel-agent-config.yaml @@ -12,8 +12,8 @@ receivers: exporters: - logging: - loglevel: debug + debug: + verbosity: detailed otlp: endpoint: "api.honeycomb.io:443" headers: @@ -30,4 +30,4 @@ service: pipelines: traces: receivers: [jaeger, otlp] - exporters: [otlp, logging] + exporters: [otlp, debug] diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 9ec372465..000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,30 +0,0 @@ -services: - integration-deps: - image: busybox - depends_on: - - server-pg - - otel-agent - server-pg: - image: postgres:16.4 - command: postgres -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all - ports: - - "5432:5432" - environment: - - POSTGRES_USER=user - - POSTGRES_PASSWORD=password - - POSTGRES_DB=pg - healthcheck: - test: [ "CMD-SHELL", "pg_isready" ] - interval: 5s - timeout: 5s - retries: 5 - otel-agent: - ports: - - "4317:4317" # OpenTelemetry receiver - image: otel/opentelemetry-collector-contrib:0.57.2 - command: [ "--config=/etc/otel-agent-config.yaml" ] - environment: - - HONEYCOMB_DATASET=${HONEYCOMB_DATASET} - - HONEYCOMB_API_KEY=${HONEYCOMB_API_KEY} - volumes: - - ./dev/otel-agent-config.yaml:/etc/otel-agent-config.yaml diff --git a/flake.lock b/flake.lock index dc34c7c9e..c2ebbeb4f 100644 --- a/flake.lock +++ b/flake.lock @@ -65,12 +65,28 @@ "type": "github" } }, + "process-compose-flake": { + "locked": { + "lastModified": 1767863885, + "narHash": "sha256-XXekPAxzbv1DmHFo3Elmj/vDnvWc1V0jdDUvM0/Wf7k=", + "owner": "Platonic-Systems", + "repo": "process-compose-flake", + "rev": "99bea96cf269cfd235833ebdf645b567069fd398", + "type": "github" + }, + "original": { + "owner": "Platonic-Systems", + "repo": "process-compose-flake", + "type": "github" + } + }, "root": { "inputs": { "advisory-db": "advisory-db", "crane": "crane", "flake-utils": "flake-utils", "nixpkgs": "nixpkgs", + "process-compose-flake": "process-compose-flake", "rust-overlay": "rust-overlay" } }, diff --git a/flake.nix b/flake.nix index 2007f7304..e6fbd28e2 100644 --- a/flake.nix +++ b/flake.nix @@ -15,6 +15,7 @@ flake = false; }; crane.url = "github:ipetkov/crane"; + process-compose-flake.url = "github:Platonic-Systems/process-compose-flake"; }; outputs = { self, @@ -23,6 +24,7 @@ rust-overlay, advisory-db, crane, + process-compose-flake, }: flake-utils.lib.eachDefaultSystem (system: let @@ -66,26 +68,178 @@ samply bacon postgresql - docker-compose - wait4x + process-compose + opentelemetry-collector-contrib bc jq - podman - podman-compose ]; + + pgPort = 5432; + otelGrpcPort = 4317; + otelHttpPort = 4318; + otelHealthPort = 13133; + pgUser = "user"; + pgPassword = "password"; + pgDatabase = "pg"; + devEnvVars = rec { - OTEL_EXPORTER_OTLP_ENDPOINT = http://localhost:4317; - DATABASE_URL = "postgres://user:password@127.0.0.1:5432/pg?sslmode=disable"; + OTEL_EXPORTER_OTLP_ENDPOINT = "http://localhost:${toString otelGrpcPort}"; + DATABASE_URL = "postgres://${pgUser}:${pgPassword}@127.0.0.1:${toString pgPort}/${pgDatabase}?sslmode=disable"; PG_CON = "${DATABASE_URL}"; }; - podman-runner = pkgs.callPackage ./nix/podman-runner.nix {}; + + # ── Postgres start helper ────────────────────────────────────────── + pg-start = pkgs.writeShellApplication { + name = "pg-start"; + runtimeInputs = [pkgs.postgresql pkgs.coreutils]; + text = '' + # Usage: pg-start + NAME="$1" PORT="$2" PGUSER="$3" DB="$4" + PGDATA="$PWD/.nix-deps/$NAME" + + mkdir -p "$PWD/.nix-deps" + + if [ ! -f "$PGDATA/PG_VERSION" ]; then + echo "[$NAME] Initializing data directory at $PGDATA..." + mkdir -p "$PGDATA" + initdb -D "$PGDATA" --username="$PGUSER" --auth=trust --no-locale -E UTF8 + { + echo "port = $PORT" + echo "unix_socket_directories = '/tmp'" + echo "listen_addresses = '127.0.0.1'" + echo "shared_preload_libraries = 'pg_stat_statements'" + echo "pg_stat_statements.track = 'all'" + } >> "$PGDATA/postgresql.conf" + fi + + if [ -f "$PGDATA/postmaster.pid" ]; then + pg_ctl -D "$PGDATA" stop -m immediate 2>/dev/null || rm -f "$PGDATA/postmaster.pid" + fi + + postgres -D "$PGDATA" -p "$PORT" -k /tmp & + PG_PID=$! + trap 'kill $PG_PID 2>/dev/null; wait $PG_PID 2>/dev/null' EXIT + + while ! pg_isready -p "$PORT" -U "$PGUSER" -h 127.0.0.1 -q 2>/dev/null; do + sleep 0.1 + done + + if [ "$DB" != "$PGUSER" ]; then + createdb -p "$PORT" -U "$PGUSER" -h 127.0.0.1 "$DB" 2>/dev/null || { + if psql -p "$PORT" -U "$PGUSER" -h 127.0.0.1 -lqt | cut -d \| -f 1 | grep -qw "$DB"; then + echo "[$NAME] Database '$DB' already exists" + else + echo "[$NAME] ERROR: Failed to create database '$DB'" >&2 + exit 1 + fi + } + fi + + echo "[$NAME] Ready on port $PORT (database: $DB)" + wait $PG_PID + ''; + }; + + setupDbDev = pkgs.writeShellApplication { + name = "setup-db-dev"; + runtimeInputs = [pkgs.sqlx-cli]; + text = '' + export DATABASE_URL="${devEnvVars.DATABASE_URL}" + cd cala-ledger + exec sqlx migrate run + ''; + }; + + startOtel = pkgs.writeShellApplication { + name = "start-otel-dev"; + runtimeInputs = [pkgs.opentelemetry-collector-contrib]; + text = '' + export HONEYCOMB_API_KEY="''${HONEYCOMB_API_KEY:-local-disabled}" + export HONEYCOMB_DATASET="''${HONEYCOMB_DATASET:-local}" + exec otelcol-contrib --config ${./dev/otel-agent-config.yaml} + ''; + }; + + # ── process-compose ─────────────────────────────────────────────── + pcLib = import process-compose-flake.lib {inherit pkgs;}; + + mkPg = { + name, + port, + user, + db, + }: { + command = "${ + pkgs.writeShellApplication { + name = "start-${name}"; + runtimeInputs = [pg-start]; + text = '' + exec pg-start ${name} ${toString port} ${user} ${db} + ''; + } + }/bin/start-${name}"; + readiness_probe = { + exec.command = "${ + pkgs.writeShellApplication { + name = "ready-${name}"; + runtimeInputs = [pkgs.postgresql]; + text = '' + exec psql -p ${toString port} -U ${user} -h 127.0.0.1 -d ${db} -c 'SELECT 1' -t -q + ''; + } + }/bin/ready-${name}"; + initial_delay_seconds = 1; + period_seconds = 1; + failure_threshold = 60; + }; + shutdown = { + signal = 2; + timeout_seconds = 10; + }; + }; + + baseProcesses = { + server-pg = mkPg { + name = "server-pg"; + port = pgPort; + user = pgUser; + db = pgDatabase; + }; + setup-db = { + command = "${setupDbDev}/bin/setup-db-dev"; + depends_on.server-pg.condition = "process_healthy"; + availability.exit_on_end = false; + shutdown = { + signal = 2; + timeout_seconds = 10; + }; + }; + otel-agent = { + command = "${startOtel}/bin/start-otel-dev"; + shutdown = { + signal = 2; + timeout_seconds = 10; + }; + }; + }; + + nix-deps-base = pcLib.makeProcessCompose { + name = "nix-deps-base"; + modules = [ + { + settings = { + log_level = "info"; + log_location = ".nix-deps/process-compose.log"; + processes = baseProcesses; + }; + } + ]; + }; perf-runner = pkgs.writeShellScriptBin "perf-runner" '' set -e export PATH="${pkgs.lib.makeBinPath [ - podman-runner.podman-compose-runner - pkgs.wait4x pkgs.sqlx-cli pkgs.coreutils pkgs.gnumake @@ -112,29 +266,15 @@ cleanup() { echo "Stopping deps..." - ${podman-runner.podman-compose-runner}/bin/podman-compose-runner down || true + ${nix-deps-base}/bin/nix-deps-base down 2>/dev/null || true } trap cleanup EXIT - echo "Starting dependencies..." - ${podman-runner.podman-compose-runner}/bin/podman-compose-runner up -d integration-deps + mkdir -p .nix-deps - echo "Waiting for PostgreSQL to be ready..." - ${pkgs.wait4x}/bin/wait4x postgresql "$DATABASE_URL" --timeout 120s - - echo "Running database migrations..." - for i in $(seq 1 30); do - if (cd cala-ledger && ${pkgs.sqlx-cli}/bin/sqlx migrate run 2>/dev/null); then - echo "Migrations complete" - break - fi - echo "Attempt $i: Database not ready, waiting..." - sleep 1 - if [ "$i" -eq 30 ]; then - echo "Database failed to become ready after 30 attempts" - cd cala-ledger && ${pkgs.sqlx-cli}/bin/sqlx migrate run - fi - done + echo "Starting dependencies via process-compose..." + ${nix-deps-base}/bin/nix-deps-base up -D + ${nix-deps-base}/bin/nix-deps-base project is-ready --wait echo "Running perf DB setup..." ${pkgs.postgresql}/bin/psql "$DATABASE_URL" -f ./cala-perf/pg-tools/setup.sql @@ -194,8 +334,6 @@ set -e export PATH="${pkgs.lib.makeBinPath [ - podman-runner.podman-compose-runner - pkgs.wait4x pkgs.sqlx-cli pkgs.cargo-nextest pkgs.coreutils @@ -210,30 +348,16 @@ cleanup() { echo "Stopping deps..." - ${podman-runner.podman-compose-runner}/bin/podman-compose-runner down || true + ${nix-deps-base}/bin/nix-deps-base down 2>/dev/null || true } trap cleanup EXIT - echo "Starting dependencies..." - ${podman-runner.podman-compose-runner}/bin/podman-compose-runner up -d integration-deps + mkdir -p .nix-deps - echo "Waiting for PostgreSQL to be ready..." - ${pkgs.wait4x}/bin/wait4x postgresql "$DATABASE_URL" --timeout 120s - - echo "Running database migrations..." - for i in $(seq 1 30); do - if (cd cala-ledger && ${pkgs.sqlx-cli}/bin/sqlx migrate run 2>/dev/null); then - echo "Migrations complete" - break - fi - echo "Attempt $i: Database not ready, waiting..." - sleep 1 - if [ "$i" -eq 30 ]; then - echo "Database failed to become ready after 30 attempts" - cd cala-ledger && ${pkgs.sqlx-cli}/bin/sqlx migrate run - fi - done + echo "Starting dependencies via process-compose..." + ${nix-deps-base}/bin/nix-deps-base up -D + ${nix-deps-base}/bin/nix-deps-base project is-ready --wait echo "Running nextest..." cargo nextest run --verbose --locked --workspace @@ -251,6 +375,13 @@ packages = { nextest = nextest-runner; perf = perf-runner; + setup-db-dev = setupDbDev; + inherit nix-deps-base; + }; + + apps.setup-db-dev = flake-utils.lib.mkApp { + drv = setupDbDev; + name = "setup-db-dev"; }; checks = { diff --git a/nix/podman-runner.nix b/nix/podman-runner.nix deleted file mode 100644 index 338dda147..000000000 --- a/nix/podman-runner.nix +++ /dev/null @@ -1,123 +0,0 @@ -{ - pkgs, - lib, - stdenv, -}: let - # Build the podman-compose runner as a derivation - podman-compose-runner = pkgs.stdenv.mkDerivation { - pname = "podman-compose-runner"; - version = "0.1.0"; - - # No source needed for a wrapper script - dontUnpack = true; - - buildInputs = with pkgs; [ - makeWrapper - ]; - - installPhase = '' - mkdir -p $out/bin - - # Create the runner script that uses podman-compose directly - cat > $out/bin/podman-compose-runner << 'EOF' - #!/usr/bin/env bash - set -e - - # On macOS, check if podman machine exists and start it if needed - if [[ "$OSTYPE" == "darwin"* ]]; then - if podman machine list --format json | jq -e '.[] | select(.Name == "podman-machine-default")' >/dev/null 2>&1; then - # Machine exists, check if it's running - if ! podman machine list --format json | jq -e '.[] | select(.Name == "podman-machine-default" and .Running == true)' >/dev/null 2>&1; then - echo "Starting podman machine..." - podman machine start - fi - else - echo "No podman machine found. Creating and starting podman-machine-default..." - podman machine init - podman machine start - fi - - # Set up minimal container configs for macOS - mkdir -p ~/.config/containers - echo 'unqualified-search-registries = ["docker.io"]' > ~/.config/containers/registries.conf - echo '{"default":[{"type":"insecureAcceptAnything"}]}' > ~/.config/containers/policy.json - else - # On Linux, setup container configs for rootless operation - echo "Using podman on Linux..." - - # Set up runtime directory for rootless containers - export XDG_RUNTIME_DIR="''${XDG_RUNTIME_DIR:-/tmp/podman-runtime-$(id -u)}" - mkdir -p "$XDG_RUNTIME_DIR" - - # Create necessary temp directories - mkdir -p /var/tmp - mkdir -p /tmp - export TMPDIR=/tmp - - # Set up minimal container configs - mkdir -p ~/.config/containers - echo 'unqualified-search-registries = ["docker.io"]' > ~/.config/containers/registries.conf - echo '{"default":[{"type":"insecureAcceptAnything"}]}' > ~/.config/containers/policy.json - - # Don't specify network backend - let podman use its default (netavark) - # But ensure iptables is available in PATH - - # Debug: Check podman version and info - echo "Checking podman installation..." - if ! podman version >/dev/null 2>&1; then - echo "ERROR: podman version failed. Output:" - podman version 2>&1 || true - fi - - # Try to get podman info for debugging - echo "Getting podman info..." - podman info 2>&1 || true - - # Check if we're in a container environment (CI) - if [[ -f /.dockerenv ]] || [[ -n "''${container:-}" ]]; then - echo "Running in container environment" - fi - - # Final check - if podman ps >/dev/null 2>&1; then - echo "Podman is working correctly" - else - echo "WARNING: podman ps test failed, but continuing anyway..." - fi - fi - - # Use podman-compose directly (it handles the socket connection internally) - exec podman-compose "$@" - EOF - - chmod +x $out/bin/podman-compose-runner - - # Wrap the script with the required dependencies - wrapProgram $out/bin/podman-compose-runner \ - --prefix PATH : ${pkgs.lib.makeBinPath ( - [ - pkgs.podman - pkgs.podman-compose - pkgs.coreutils - pkgs.bash - pkgs.jq - ] - ++ pkgs.lib.optionals pkgs.stdenv.isLinux [ - pkgs.fuse-overlayfs - pkgs.iptables - pkgs.netavark - pkgs.aardvark-dns - ] - )} - ''; - - meta = with pkgs.lib; { - description = "Podman-compose runner that auto-manages podman machine on macOS"; - license = licenses.mit; - platforms = platforms.all; - }; - }; -in { - # Default package is the full runner with machine management - podman-compose-runner = podman-compose-runner; -} From 31460dc426075a8a2cf5ee411c1aa3d76677e4a9 Mon Sep 17 00:00:00 2001 From: vindard <17693119+vindard@users.noreply.github.com> Date: Wed, 27 May 2026 10:01:37 -0400 Subject: [PATCH 3/3] chore: add readiness probe to otel-agent process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the only gap in per-service readiness coverage: without a probe, process-compose marks otel-agent as Running but never Ready, so `is-ready --wait` hangs forever if the collector fails to start (e.g. a deprecated config option that decoded fine until otelcol 0.144). With the probe, PC fails the process after 60s and the wait exits non-zero — fast, self-describing failure instead of CI timeout. The probe targets the health_check extension already enabled in dev/otel-agent-config.yaml (default :13133/). --- flake.nix | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/flake.nix b/flake.nix index e6fbd28e2..944f4369c 100644 --- a/flake.nix +++ b/flake.nix @@ -216,6 +216,20 @@ }; otel-agent = { command = "${startOtel}/bin/start-otel-dev"; + readiness_probe = { + exec.command = "${ + pkgs.writeShellApplication { + name = "ready-otel"; + runtimeInputs = [pkgs.curl]; + text = '' + exec curl -fsS "http://127.0.0.1:${toString otelHealthPort}/" + ''; + } + }/bin/ready-otel"; + initial_delay_seconds = 2; + period_seconds = 2; + failure_threshold = 30; + }; shutdown = { signal = 2; timeout_seconds = 10;