From 79fb71af55b95247bf697d4fa3b1c9df2e2089bd Mon Sep 17 00:00:00 2001 From: joshua-spacetime Date: Thu, 7 May 2026 11:49:54 -0700 Subject: [PATCH 1/7] Update keynote benchmark to run for 60s by default --- templates/keynote-2/DEVELOP.md | 4 ++-- templates/keynote-2/README.md | 6 +++--- templates/keynote-2/src/opts.ts | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/templates/keynote-2/DEVELOP.md b/templates/keynote-2/DEVELOP.md index d99c1ffedb4..f3211094255 100644 --- a/templates/keynote-2/DEVELOP.md +++ b/templates/keynote-2/DEVELOP.md @@ -27,7 +27,7 @@ The script will: **Options:** -- `--seconds N` - Benchmark duration (default: 10) +- `--seconds N` - Benchmark duration (default: 60) - `--concurrency N` - Concurrent connections (default: 50) - `--alpha N` - Contention level (default: 1.5) - `--systems a,b,c` - Systems to compare (default: convex,spacetimedb) @@ -205,7 +205,7 @@ From `src/cli.ts`: - **`--seconds N`** - Duration of the benchmark in seconds - - Default: `10` + - Default: `60` - **`--concurrency N`** - Number of workers / in-flight operations diff --git a/templates/keynote-2/README.md b/templates/keynote-2/README.md index 9f3cbc9e70f..3eef2aec448 100644 --- a/templates/keynote-2/README.md +++ b/templates/keynote-2/README.md @@ -20,7 +20,7 @@ The demo compares SpacetimeDB and Convex by default, since both are easy for any ## Results Summary -All tests use 50 concurrent connections with a transfer workload (read-modify-write transaction between two accounts). +All tests run for 60 seconds and use 50 concurrent connections with a transfer workload (read-modify-write transaction between two accounts). | System | TPS (~0% Contention) | TPS (~80% Contention) | | --------------------------------- | -------------------- | --------------------- | @@ -87,10 +87,10 @@ This is a classic read-modify-write workload that tests transactional integrity ### Test Command ```bash -docker compose run --rm bench -- --seconds 10 --concurrency 50 --alpha XX --connectors YY +docker compose run --rm bench -- --seconds 60 --concurrency 50 --alpha XX --connectors YY ``` -- `--seconds 10`: Duration of benchmark run +- `--seconds 60`: Duration of benchmark run - `--concurrency 50`: Number of concurrent client connections - `--alpha 0`: ~0% contention (uniform account distribution) - `--alpha 1.5`: ~80% contention (Zipf distribution concentrating on hot accounts) diff --git a/templates/keynote-2/src/opts.ts b/templates/keynote-2/src/opts.ts index 98474c53082..ecd14538eb7 100644 --- a/templates/keynote-2/src/opts.ts +++ b/templates/keynote-2/src/opts.ts @@ -301,7 +301,7 @@ export function parseDemoOptions(argv: string[] = process.argv): DemoOptions { return { ...runtimeOptions, - seconds: options.seconds ?? 10, + seconds: options.seconds ?? 60, concurrency: options.concurrency ?? 50, alpha: options.alpha ?? 1.5, systems: @@ -374,7 +374,7 @@ export function parseBenchOptions(argv: string[] = process.argv): BenchOptions { return { ...runtimeOptions, testName: args[0] ?? defaultBenchTestName, - seconds: options.seconds ?? 10, + seconds: options.seconds ?? 60, concurrency: contentionTests?.concurrency ?? options.concurrency ?? 50, alpha: concurrencyTests?.alpha ?? options.alpha ?? 1.5, From 54d17a4b76a9d01a83e1c2da41e2e36b3aae1f47 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Thu, 7 May 2026 17:35:43 -0400 Subject: [PATCH 2/7] Updates - bump convex to latest - update numbers - update init_convex due to new limits on writes/second on local dev - update compose to default settings (*except* default_transaction_isolation=serializable for pg) --- templates/keynote-2/README.md | 37 +++++++++++---------- templates/keynote-2/convex-app/package.json | 2 +- templates/keynote-2/docker-compose.yml | 16 ++------- templates/keynote-2/package.json | 2 +- templates/keynote-2/src/init/init_convex.ts | 3 +- 5 files changed, 26 insertions(+), 34 deletions(-) diff --git a/templates/keynote-2/README.md b/templates/keynote-2/README.md index 3eef2aec448..578cc9d3a23 100644 --- a/templates/keynote-2/README.md +++ b/templates/keynote-2/README.md @@ -20,21 +20,21 @@ The demo compares SpacetimeDB and Convex by default, since both are easy for any ## Results Summary -All tests run for 60 seconds and use 50 concurrent connections with a transfer workload (read-modify-write transaction between two accounts). +All tests run for 60 seconds with 50 concurrent connections, with a transfer workload (read-modify-write transaction between two accounts). | System | TPS (~0% Contention) | TPS (~80% Contention) | | --------------------------------- | -------------------- | --------------------- | | SpacetimeDB (TypeScript Module) | | 307,074 | | SpacetimeDB (Rust Module) | | 265,542 | -| SQLite + Node HTTP + Drizzle | | 3,236 | -| Bun + Drizzle + Postgres | 7,115 | 2,074 | -| Postgres + Node HTTP + Drizzle | 6,429 | 2,798 | -| Supabase + Node HTTP + Drizzle | 6,310 | 1,268 | -| CockroachDB + Node HTTP + Drizzle | 5,129 | 197 | -| PlanetScale + Node HTTP + Drizzle | 477 | 30 | -| Convex | 438 | 58 | +| SQLite + Node HTTP + Drizzle | 3,095 | 3,186 | +| Bun + Drizzle + Postgres | 10,736 | 2,817 | +| Supabase + Node HTTP + Drizzle | 7,299 | 2,582 | +| Postgres + Node HTTP + Drizzle | 9,946 | 1,111 | +| Convex (self-hosted local) | 1,218 | 225 | +| PlanetScale + Node HTTP + Drizzle | - | 203 | +| CockroachDB + Node HTTP + Drizzle | 3,931 | 142 | -**Key Finding:** SpacetimeDB reaches hundreds of thousands of TPS for the transfer workload, while the best non-SpacetimeDB result shown here is SQLite RPC at 3,236 TPS. Traditional databases also suffer significant degradation under high contention (CockroachDB drops 96%). +**Key Finding:** SpacetimeDB reaches hundreds of thousands of TPS for the transfer workload, while the best non-SpacetimeDB result shown here is SQLite at 3,186 TPS. Traditional databases also suffer significant degradation under high contention (CockroachDB drops 96%). ### Contention Impact @@ -44,13 +44,9 @@ The chart above shows TPS vs Zipf Alpha (contention level). Higher alpha values ## Methodology -All systems were tested with **out-of-the-box default settings** - no custom tuning, no configuration optimization. This reflects what developers experience when they first adopt these technologies. +All systems were tested with **out-of-the-box default settings**: no custom tuning, no configuration optimization. This reflects what developers experience when they first adopt these technologies. -For cloud services, we tested paid tiers to give them their best chance: - -- **PlanetScale**: PS-2560 (32 vCPUs, 256 GB RAM), single node, us-central1. -- **Supabase**: Pro tier -- **Convex**: Pro tier +Postgres (and Bun, which uses the same Postgres instance) is configured with `default_transaction_isolation = 'serializable'`. The reported SpacetimeDB module results were run against a 5-way replicated cluster rather than a single standalone node. @@ -86,8 +82,13 @@ This is a classic read-modify-write workload that tests transactional integrity ### Test Command +The numbers in the table above were collected by running each connector directly with `pnpm`: + ```bash -docker compose run --rm bench -- --seconds 60 --concurrency 50 --alpha XX --connectors YY +pnpm install +pnpm run prep # seed all backing databases once +pnpm run bench test-1 --seconds 60 --concurrency 50 --alpha 0 --connectors # uncontended +pnpm run bench test-1 --seconds 60 --concurrency 50 --alpha 1.5 --connectors # ~80% contention ``` - `--seconds 60`: Duration of benchmark run @@ -96,6 +97,8 @@ docker compose run --rm bench -- --seconds 60 --concurrency 50 --alpha XX --conn - `--alpha 1.5`: ~80% contention (Zipf distribution concentrating on hot accounts) - `--stdb-compression none|gzip`: SpacetimeDB client compression mode (default: `none`) +The Docker workflow (`docker compose run --rm bench -- ...`) produces equivalent numbers. + ### Hardware Configuration **Server Machine (Variant A - PhoenixNAP):** @@ -160,7 +163,7 @@ SpacetimeDB supports `withConfirmedReads` mode which ensures transactions are du ### Cloud vs Local Results -PlanetScale results (~477 TPS) demonstrate the **significant impact of cloud database latency**. When the database is accessed over the network (even within the same cloud region), round-trip latency dominates performance. This is why SpacetimeDB's colocated architecture provides such dramatic improvements. +PlanetScale results (203 TPS under high contention) demonstrate the **significant impact of cloud database latency**. When the database is accessed over the network (even within the same cloud region), round-trip latency dominates performance. This is why SpacetimeDB's colocated architecture provides such dramatic improvements. ## Systems Tested diff --git a/templates/keynote-2/convex-app/package.json b/templates/keynote-2/convex-app/package.json index eebdd65afc6..e5997e51d0f 100644 --- a/templates/keynote-2/convex-app/package.json +++ b/templates/keynote-2/convex-app/package.json @@ -7,7 +7,7 @@ }, "dependencies": { "@convex-dev/sharded-counter": "^0.2.0", - "convex": "^1.29.3", + "convex": "^1.37.0", "convex-helpers": "^0.1.0" } } diff --git a/templates/keynote-2/docker-compose.yml b/templates/keynote-2/docker-compose.yml index e86735fd13d..d4d07dd8ab6 100644 --- a/templates/keynote-2/docker-compose.yml +++ b/templates/keynote-2/docker-compose.yml @@ -7,19 +7,7 @@ POSTGRES_DB: ${POSTGRES_DB} ports: - "5432:5432" - # command: > - # -c fsync=on - # -c synchronous_commit=on - # -c shared_buffers=2GB - # -c work_mem=64MB - # -c max_connections=1000 - command: > # faster setup - -c fsync=on - -c synchronous_commit=off - -c shared_buffers=8GB - -c work_mem=64MB - -c max_connections=10000 - -c default_transaction_isolation=serializable + command: -c default_transaction_isolation=serializable healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] interval: 2s @@ -31,7 +19,7 @@ crdb: image: cockroachdb/cockroach:latest - command: start-single-node --insecure --max-sql-memory=.25 --cache=.5 + command: start-single-node --insecure ports: - "26257:26257" - "8082:8080" diff --git a/templates/keynote-2/package.json b/templates/keynote-2/package.json index 05157e7d59e..e7278fdb908 100644 --- a/templates/keynote-2/package.json +++ b/templates/keynote-2/package.json @@ -20,7 +20,7 @@ "@types/pg": "^8.15.6", "@types/sql.js": "^1.4.9", "bun-types": "^1.3.2", - "convex": "^1.29.0", + "convex": "^1.37.0", "dotenv": "^17.2.3", "node-gyp": "^12.1.0", "pg": "^8.16.3", diff --git a/templates/keynote-2/src/init/init_convex.ts b/templates/keynote-2/src/init/init_convex.ts index 5fcb630294c..1fae6e0c786 100644 --- a/templates/keynote-2/src/init/init_convex.ts +++ b/templates/keynote-2/src/init/init_convex.ts @@ -50,7 +50,7 @@ export async function initConvex(config: ConvexInitConfig) { } // Max ~16k writes per function; keep a safety margin - const CHUNK = 10_000; + const CHUNK = 7_000; console.log( `[convex] seeding ${accounts} accounts in chunks of ${CHUNK} (initial=${initialBalance})`, ); @@ -65,6 +65,7 @@ export async function initConvex(config: ConvexInitConfig) { count, initial: initialBalance, }); + await new Promise(r => setTimeout(r, 500)); } console.log('[convex] seed complete.'); From 94023cb288cfe83cacc51024c8d8e35d0555dbee Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Thu, 7 May 2026 17:38:53 -0400 Subject: [PATCH 3/7] Remove separate client in readme --- templates/keynote-2/README.md | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/templates/keynote-2/README.md b/templates/keynote-2/README.md index 578cc9d3a23..497ead6754a 100644 --- a/templates/keynote-2/README.md +++ b/templates/keynote-2/README.md @@ -101,31 +101,9 @@ The Docker workflow (`docker compose run --rm bench -- ...`) produces equivalent ### Hardware Configuration -**Server Machine (Variant A - PhoenixNAP):** +**Server Machine:** -- s3.c3.medium bare metal instance - Intel i9-14900k 24 cores (32 threads), 128GB DDR5 Memory, OS: Ubuntu 24.04 - -**Server Machine (Variant B - Google Cloud):** - -- c4-standard-32-lssd (32 vCPUs, 120 GB Memory) OS: Ubuntu 24.04 -- RAID 0 on 5 Local SSDs -- Region: us-central1 - -**Client Machine:** - -- c4-standard-32 (32 vCPUs, 120 GB Memory) OS: Ubuntu 24.04 -- Region: us-central1 -- Runs on a **separate machine** from the server - -**Note:** All services (databases, web servers, benchmark runner) except Convex local dev backend run in the same Docker environment on the server machine. - -### Why Separate Client Machines? - -Running clients on separate machines ensures: - -- Network round-trip latency is measured (realistic production scenario) -- Client CPU/memory doesn't compete with server resources -- Results reflect actual deployment conditions +- PhoenixNAP s3.c3.medium bare metal instance - Intel i9-14900k 24 cores (32 threads), 128GB DDR5 Memory, OS: Ubuntu 24.04 ### Account Seeding From 283e8129662c09da90d65603c3264a6d0156e57b Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Thu, 7 May 2026 17:42:46 -0400 Subject: [PATCH 4/7] Update README.md --- templates/keynote-2/README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/templates/keynote-2/README.md b/templates/keynote-2/README.md index 497ead6754a..c47145b4b6c 100644 --- a/templates/keynote-2/README.md +++ b/templates/keynote-2/README.md @@ -44,9 +44,7 @@ The chart above shows TPS vs Zipf Alpha (contention level). Higher alpha values ## Methodology -All systems were tested with **out-of-the-box default settings**: no custom tuning, no configuration optimization. This reflects what developers experience when they first adopt these technologies. - -Postgres (and Bun, which uses the same Postgres instance) is configured with `default_transaction_isolation = 'serializable'`. +All systems were tested with **out-of-the-box default settings**, with one exception: Postgres (and Bun, which uses the same Postgres instance) is configured with `default_transaction_isolation = 'serializable'` for an apples-to-apples comparison. No other custom tuning or configuration optimization was applied. The reported SpacetimeDB module results were run against a 5-way replicated cluster rather than a single standalone node. From ed552b1683e2c05bb3c08e9596639da3e0b17091 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Thu, 7 May 2026 17:53:02 -0400 Subject: [PATCH 5/7] Update README.md --- templates/keynote-2/README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/templates/keynote-2/README.md b/templates/keynote-2/README.md index c47145b4b6c..6221526f983 100644 --- a/templates/keynote-2/README.md +++ b/templates/keynote-2/README.md @@ -36,12 +36,6 @@ All tests run for 60 seconds with 50 concurrent connections, with a transfer wor **Key Finding:** SpacetimeDB reaches hundreds of thousands of TPS for the transfer workload, while the best non-SpacetimeDB result shown here is SQLite at 3,186 TPS. Traditional databases also suffer significant degradation under high contention (CockroachDB drops 96%). -### Contention Impact - -![Contention Chart](./contention-chart.png) - -The chart above shows TPS vs Zipf Alpha (contention level). Higher alpha values concentrate more transactions on fewer "hot" accounts, increasing contention. SpacetimeDB maintains consistent performance regardless of contention level, while traditional database architectures show significant degradation. - ## Methodology All systems were tested with **out-of-the-box default settings**, with one exception: Postgres (and Bun, which uses the same Postgres instance) is configured with `default_transaction_isolation = 'serializable'` for an apples-to-apples comparison. No other custom tuning or configuration optimization was applied. From 07850c6803ded9f016c03d8d4b7fe1f942041001 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Thu, 7 May 2026 19:48:29 -0400 Subject: [PATCH 6/7] Update README.md --- templates/keynote-2/README.md | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/templates/keynote-2/README.md b/templates/keynote-2/README.md index 6221526f983..dac961b5923 100644 --- a/templates/keynote-2/README.md +++ b/templates/keynote-2/README.md @@ -22,19 +22,20 @@ The demo compares SpacetimeDB and Convex by default, since both are easy for any All tests run for 60 seconds with 50 concurrent connections, with a transfer workload (read-modify-write transaction between two accounts). -| System | TPS (~0% Contention) | TPS (~80% Contention) | -| --------------------------------- | -------------------- | --------------------- | -| SpacetimeDB (TypeScript Module) | | 307,074 | -| SpacetimeDB (Rust Module) | | 265,542 | -| SQLite + Node HTTP + Drizzle | 3,095 | 3,186 | -| Bun + Drizzle + Postgres | 10,736 | 2,817 | -| Supabase + Node HTTP + Drizzle | 7,299 | 2,582 | -| Postgres + Node HTTP + Drizzle | 9,946 | 1,111 | -| Convex (self-hosted local) | 1,218 | 225 | -| PlanetScale + Node HTTP + Drizzle | - | 203 | -| CockroachDB + Node HTTP + Drizzle | 3,931 | 142 | - -**Key Finding:** SpacetimeDB reaches hundreds of thousands of TPS for the transfer workload, while the best non-SpacetimeDB result shown here is SQLite at 3,186 TPS. Traditional databases also suffer significant degradation under high contention (CockroachDB drops 96%). +Each cell shows **mean TPS ± sample standard deviation** across 3 × 60-second runs. Cells where the standard deviation approaches or exceeds the mean (e.g. CockroachDB and Convex at ~80% contention) indicate that the system's throughput varies substantially between runs. + +| System | Mean TPS (~0% Contention) | Mean TPS (~80% Contention) | +| --------------------------------- | ------------------------- | -------------------------- | +| SpacetimeDB (TypeScript Module) | | 307,074 | +| SpacetimeDB (Rust Module) | | 265,542 | +| SQLite + Node HTTP + Drizzle | 3,081 ± 15 | 3,169 ± 16 | +| Bun + Drizzle + Postgres | 10,582 ± 7 | 2,754 ± 5 | +| Supabase + Node HTTP + Drizzle | 7,116 ± 161 | 2,581 ± 2 | +| Postgres + Node HTTP + Drizzle | 9,425 ± 25 | 1,087 ± 5 | +| CockroachDB + Node HTTP + Drizzle | 3,933 ± 28 | 145 ± 168 | +| Convex (self-hosted local) | 1,210 ± 47 | 130 ± 98 | + +**Key Finding:** SpacetimeDB reaches hundreds of thousands of TPS for the transfer workload, while the best non-SpacetimeDB result shown here is SQLite at 3,169 TPS. Traditional databases also suffer significant degradation under high contention (CockroachDB drops 96%). ## Methodology From 6758146561645202f6f215c4d5968469a8d831c7 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Thu, 7 May 2026 20:11:32 -0400 Subject: [PATCH 7/7] Update README.md --- templates/keynote-2/README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/templates/keynote-2/README.md b/templates/keynote-2/README.md index dac961b5923..9308a3dfa35 100644 --- a/templates/keynote-2/README.md +++ b/templates/keynote-2/README.md @@ -22,18 +22,18 @@ The demo compares SpacetimeDB and Convex by default, since both are easy for any All tests run for 60 seconds with 50 concurrent connections, with a transfer workload (read-modify-write transaction between two accounts). -Each cell shows **mean TPS ± sample standard deviation** across 3 × 60-second runs. Cells where the standard deviation approaches or exceeds the mean (e.g. CockroachDB and Convex at ~80% contention) indicate that the system's throughput varies substantially between runs. - -| System | Mean TPS (~0% Contention) | Mean TPS (~80% Contention) | -| --------------------------------- | ------------------------- | -------------------------- | -| SpacetimeDB (TypeScript Module) | | 307,074 | -| SpacetimeDB (Rust Module) | | 265,542 | -| SQLite + Node HTTP + Drizzle | 3,081 ± 15 | 3,169 ± 16 | -| Bun + Drizzle + Postgres | 10,582 ± 7 | 2,754 ± 5 | -| Supabase + Node HTTP + Drizzle | 7,116 ± 161 | 2,581 ± 2 | -| Postgres + Node HTTP + Drizzle | 9,425 ± 25 | 1,087 ± 5 | -| CockroachDB + Node HTTP + Drizzle | 3,933 ± 28 | 145 ± 168 | -| Convex (self-hosted local) | 1,210 ± 47 | 130 ± 98 | +Each cell shows **mean TPS ± sample standard deviation** across 3 × 60-second runs, with the sample variance in parentheses. Cells where the standard deviation approaches or exceeds the mean (e.g. CockroachDB and Convex at ~80% contention) indicate that the system's throughput varies substantially between runs. + +| System | Mean TPS ± σ (Var) (~0% Contention) | Mean TPS ± σ (Var) (~80% Contention) | +| --------------------------------- | ----------------------------------- | ------------------------------------ | +| SpacetimeDB (TypeScript Module) | | 307,074 | +| SpacetimeDB (Rust Module) | | 265,542 | +| SQLite + Node HTTP + Drizzle | 3,081 ± 15 (224) | 3,169 ± 16 (242) | +| Bun + Drizzle + Postgres | 10,582 ± 7 (53) | 2,754 ± 5 (28) | +| Supabase + Node HTTP + Drizzle | 7,116 ± 161 (25,763) | 2,581 ± 2 (3) | +| Postgres + Node HTTP + Drizzle | 9,425 ± 25 (644) | 1,087 ± 5 (21) | +| CockroachDB + Node HTTP + Drizzle | 3,933 ± 28 (762) | 145 ± 168 (28,157) | +| Convex (self-hosted local) | 1,210 ± 47 (2,224) | 130 ± 98 (9,675) | **Key Finding:** SpacetimeDB reaches hundreds of thousands of TPS for the transfer workload, while the best non-SpacetimeDB result shown here is SQLite at 3,169 TPS. Traditional databases also suffer significant degradation under high contention (CockroachDB drops 96%).