diff --git a/.autoloop/programs/perf-comparison/program.md b/.autoloop/programs/perf-comparison/program.md new file mode 100644 index 00000000..c1aec206 --- /dev/null +++ b/.autoloop/programs/perf-comparison/program.md @@ -0,0 +1,74 @@ +--- +schedule: every 6h +--- + +# Performance Comparison: tsb (TypeScript) vs pandas (Python) + +## Goal + +Systematically benchmark every tsb function against its pandas equivalent, one function per iteration. Each iteration picks a function that has not yet been benchmarked, writes a matching performance test for both tsb (TypeScript/Bun) and pandas (Python), runs both, and records the timing results. The benchmark results are displayed on the playground pages doc site. + +This is an open-ended program — it runs continuously, always adding the next benchmark comparison. + +### How each iteration works + +1. **Read existing benchmarks** — check `benchmarks/tsb/` and `benchmarks/pandas/` to see which functions are already benchmarked. +2. **Pick ONE function** from `src/` that has no benchmark yet. Prioritize core operations (Series, DataFrame, GroupBy, etc.). +3. **Write a TypeScript benchmark** in `benchmarks/tsb/bench_{function}.ts` that: + - Creates a realistic dataset (e.g. 100,000 rows) + - Runs the operation in a tight loop (warm-up + measured iterations) + - Outputs JSON: `{"function": "...", "mean_ms": ..., "iterations": ..., "total_ms": ...}` +4. **Write a matching Python benchmark** in `benchmarks/pandas/bench_{function}.py` that: + - Creates the same dataset as the TypeScript version + - Runs the same operation with the same loop structure + - Outputs the same JSON format +5. **Run both benchmarks** via `benchmarks/run_benchmarks.sh` and capture results. +6. **Update `benchmarks/results.json`** with the new timing data. +7. **Update `playground/benchmarks.html`** to display the new function's comparison metrics. + +### Key constraints + +- **Matching datasets** — both benchmarks must use identical data (same size, same values where possible). +- **Fair comparison** — same number of warm-up and measured iterations for both. +- **JSON output** — every benchmark script must output a single JSON line to stdout. +- **No modifications to `src/`** — benchmark code is separate from library code. +- **Python environment** — install pandas via pip if not present. + +## Target + +Only modify these files: +- `benchmarks/**` — benchmark scripts and results +- `playground/benchmarks.html` — performance comparison playground page +- `playground/index.html` — add/update link to benchmarks page + +Do NOT modify: +- `src/**` — library source code +- `tests/**` — test files +- `README.md` — read-only +- `.autoloop/programs/**` — program definitions (except this file's code/ dir) +- `.github/workflows/autoloop*` — autoloop workflow files + +## Evaluation + +```bash +# Set up Python environment if needed +if ! command -v python3 &>/dev/null; then + echo "Python3 not found, skipping" +fi +pip3 install pandas --quiet 2>/dev/null || true + +# Count the number of benchmark pairs (functions with both TS and Python benchmarks) +ts_benchmarks=$(ls benchmarks/tsb/bench_*.ts 2>/dev/null | wc -l | tr -d ' ') +py_benchmarks=$(ls benchmarks/pandas/bench_*.py 2>/dev/null | wc -l | tr -d ' ') + +# The metric is the minimum of the two (both must exist for a complete benchmark) +if [ "$ts_benchmarks" -lt "$py_benchmarks" ]; then + count=$ts_benchmarks +else + count=$py_benchmarks +fi + +echo "{\"benchmarked_functions\": ${count:-0}}" +``` + +The metric is `benchmarked_functions`. **Higher is better.** diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index a7ede9cd..127a90d6 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -36,6 +36,13 @@ jobs: - name: Bundle TypeScript compiler for offline playground run: cp node_modules/typescript/lib/typescript.js ./playground/dist/typescript.js + - name: Copy benchmark results to playground + run: | + mkdir -p ./playground/benchmarks + if [ -f benchmarks/results.json ]; then + cp benchmarks/results.json ./playground/benchmarks/results.json + fi + - name: Setup Python uses: actions/setup-python@v5 with: diff --git a/benchmarks/pandas/bench_series_creation.py b/benchmarks/pandas/bench_series_creation.py new file mode 100644 index 00000000..c27fcf87 --- /dev/null +++ b/benchmarks/pandas/bench_series_creation.py @@ -0,0 +1,47 @@ +""" +Benchmark: Series creation + +Creates a Series from a large numeric array and measures the time. +Outputs JSON: {"function": "series_creation", "mean_ms": ..., "iterations": ..., "total_ms": ...} +""" + +import json +import time + +import pandas as pd + +SIZE = 100_000 +WARMUP = 5 +ITERATIONS = 50 + + +def generate_data(n: int) -> "list[float]": + """Generate a deterministic numeric array of the given size.""" + return [i * 1.1 + 0.5 for i in range(n)] + + +data = generate_data(SIZE) + +# Warm-up +for _ in range(WARMUP): + pd.Series(list(data)) + +# Measured runs +times: "list[float]" = [] +for _ in range(ITERATIONS): + start = time.perf_counter() + pd.Series(list(data)) + end = time.perf_counter() + times.append((end - start) * 1000) # convert to ms + +total_ms = sum(times) +mean_ms = total_ms / ITERATIONS + +result = { + "function": "series_creation", + "mean_ms": round(mean_ms, 3), + "iterations": ITERATIONS, + "total_ms": round(total_ms, 3), +} + +print(json.dumps(result)) diff --git a/benchmarks/results.json b/benchmarks/results.json new file mode 100644 index 00000000..7d1fa6ec --- /dev/null +++ b/benchmarks/results.json @@ -0,0 +1 @@ +{ "benchmarks": [], "timestamp": null } diff --git a/benchmarks/run_benchmarks.sh b/benchmarks/run_benchmarks.sh new file mode 100755 index 00000000..0f800de0 --- /dev/null +++ b/benchmarks/run_benchmarks.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# +# Run all tsb (TypeScript) and pandas (Python) benchmarks and collect results. +# +# Usage: ./benchmarks/run_benchmarks.sh +# +# Outputs: benchmarks/results.json with all benchmark results +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +# Ensure Python and pandas are available +if ! command -v python3 &>/dev/null; then + echo "ERROR: python3 is required but not found" >&2 + exit 1 +fi + +python3 -c "import pandas" 2>/dev/null || { + echo "Installing pandas..." + pip3 install pandas --quiet +} + +# Ensure Bun is available +if ! command -v bun &>/dev/null; then + echo "ERROR: bun is required but not found" >&2 + exit 1 +fi + +# Collect results +results='{"benchmarks": [], "timestamp": "'$(date -u +"%Y-%m-%dT%H:%M:%SZ")'"}' + +echo "=== Running Performance Benchmarks ===" +echo "" + +# Find all TypeScript benchmark files +for ts_bench in "$SCRIPT_DIR"/tsb/bench_*.ts; do + [ -f "$ts_bench" ] || continue + bench_name=$(basename "$ts_bench" .ts | sed 's/^bench_//') + + # Check for matching Python benchmark + py_bench="$SCRIPT_DIR/pandas/bench_${bench_name}.py" + if [ ! -f "$py_bench" ]; then + echo "SKIP: $bench_name (no matching Python benchmark)" + continue + fi + + echo "--- Benchmarking: $bench_name ---" + + # Run TypeScript benchmark + echo " Running tsb (TypeScript)..." + ts_result=$(cd "$REPO_ROOT" && bun run "$ts_bench" 2>/dev/null) || { + echo " ERROR: TypeScript benchmark failed" + continue + } + echo " tsb result: $ts_result" + + # Run Python benchmark + echo " Running pandas (Python)..." + py_result=$(cd "$REPO_ROOT" && python3 "$py_bench" 2>/dev/null) || { + echo " ERROR: Python benchmark failed" + continue + } + echo " pandas result: $py_result" + + # Extract mean_ms from both + ts_mean=$(echo "$ts_result" | python3 -c "import sys, json; d=json.load(sys.stdin); print(d['mean_ms'])" 2>/dev/null) || { + echo " ERROR: could not parse tsb benchmark result" + continue + } + py_mean=$(echo "$py_result" | python3 -c "import sys, json; d=json.load(sys.stdin); print(d['mean_ms'])" 2>/dev/null) || { + echo " ERROR: could not parse pandas benchmark result" + continue + } + + # Calculate ratio (tsb / pandas) — < 1.0 means tsb is faster + ratio=$(python3 -c " +ts, py = $ts_mean, $py_mean +if py <= 0: + print('null') +else: + print(round(ts / py, 3)) +") + if [ "$ratio" = "null" ]; then + echo " ERROR: pandas mean_ms is zero, cannot compute ratio" + continue + fi + + echo " Ratio (tsb/pandas): ${ratio}x" + echo "" + + # Add to results JSON + results=$(echo "$results" | python3 -c " +import sys, json +data = json.load(sys.stdin) +data['benchmarks'].append({ + 'function': '$bench_name', + 'tsb': $ts_result, + 'pandas': $py_result, + 'ratio': $ratio +}) +print(json.dumps(data, indent=2)) +") +done + +# Write results +echo "$results" > "$SCRIPT_DIR/results.json" +echo "=== Results written to benchmarks/results.json ===" +echo "" + +# Summary +echo "=== Summary ===" +echo "$results" | python3 -c " +import sys, json +data = json.load(sys.stdin) +benchmarks = data.get('benchmarks', []) +if not benchmarks: + print('No benchmarks found.') +else: + print(f'Functions benchmarked: {len(benchmarks)}') + for b in benchmarks: + fn = b['function'] + ts = b['tsb']['mean_ms'] + py = b['pandas']['mean_ms'] + ratio = b['ratio'] + faster = 'tsb' if ratio < 1 else 'pandas' + print(f' {fn}: tsb={ts}ms, pandas={py}ms, ratio={ratio}x ({faster} is faster)') +" diff --git a/benchmarks/tsb/bench_series_creation.ts b/benchmarks/tsb/bench_series_creation.ts new file mode 100644 index 00000000..c7b4e145 --- /dev/null +++ b/benchmarks/tsb/bench_series_creation.ts @@ -0,0 +1,49 @@ +/** + * Benchmark: Series creation + * + * Creates a Series from a large numeric array and measures the time. + * Outputs JSON: {"function": "series_creation", "mean_ms": ..., "iterations": ..., "total_ms": ...} + */ + +import { Series } from "../../src/index.ts"; + +const SIZE = 100_000; +const WARMUP = 5; +const ITERATIONS = 50; + +/** Generate a deterministic numeric array of the given size. */ +function generateData(n: number): readonly number[] { + const arr: number[] = []; + for (let i = 0; i < n; i++) { + arr.push(i * 1.1 + 0.5); + } + return arr; +} + +const data = generateData(SIZE); + +// Warm-up +for (let i = 0; i < WARMUP; i++) { + new Series({ data: [...data] }); +} + +// Measured runs +const times: number[] = []; +for (let i = 0; i < ITERATIONS; i++) { + const start = performance.now(); + new Series({ data: [...data] }); + const end = performance.now(); + times.push(end - start); +} + +const totalMs = times.reduce((a, b) => a + b, 0); +const meanMs = totalMs / ITERATIONS; + +const result = { + function: "series_creation", + mean_ms: Math.round(meanMs * 1000) / 1000, + iterations: ITERATIONS, + total_ms: Math.round(totalMs * 1000) / 1000, +}; + +console.log(JSON.stringify(result)); diff --git a/playground/benchmarks.html b/playground/benchmarks.html new file mode 100644 index 00000000..6b5dde65 --- /dev/null +++ b/playground/benchmarks.html @@ -0,0 +1,345 @@ + + + + + + tsb — Performance Benchmarks: tsb vs pandas + + + + ← back to index +

⚡ Performance Benchmarks

+

+ Side-by-side performance comparison of tsb (TypeScript/Bun) vs + pandas (Python). Each function is benchmarked with identical datasets + and the same number of iterations. +

+ +
+
+
+ tsb (TypeScript) +
+
+
+ pandas (Python) +
+
+ + + + + +
+ + + + + + + + + + + + + + + + + + +
+

📐 Methodology

+

Each benchmark follows a consistent protocol:

+ +
+ +
+

🤖 About

+

+ These benchmarks are generated automatically by the Autoloop + perf-comparison program. Each iteration adds a new function + comparison. Results are updated on every accepted iteration and deployed + to this page. +

+
+ + + + + + diff --git a/playground/index.html b/playground/index.html index 48bfbcb9..b5499765 100644 --- a/playground/index.html +++ b/playground/index.html @@ -266,6 +266,17 @@

+

Performance

+
+
+

⚡ Benchmarks

+

Side-by-side performance comparison of tsb (TypeScript/Bun) vs pandas (Python). Timing metrics for each function.

+
🏗️ In Progress
+
+
+