Add database integration

Makefile

@@ -15,7 +15,8 @@ help:
 	@echo "  make init-gcp         - Initialize GCP project (APIs, registry, bucket)"
 	@echo ""
 	@echo "Development:"
-	@echo "  make dev              - Start all services in development mode"
+	@echo "  make dev              - Start all services in development mode (Docker)"
+	@echo "  make dev-local        - Start services using local venvs (no Docker)"
 	@echo "  make up [service=x]   - Start specific service or all services"
 	@echo "  make down             - Stop all services"
 	@echo "  make logs [service=x] - Show logs for service"
@@ -61,6 +62,29 @@ setup:
 
 # Development commands
 dev:
+	@echo "Starting PolicyEngine services..."
+	docker-compose -f deployment/docker-compose.yml up --build -d
+	@echo ""
+	@echo "Starting Supabase (requires Supabase CLI installed locally)..."
+	@if command -v supabase >/dev/null 2>&1; then \
+		cd projects/supabase && supabase start; \
+		echo ""; \
+		echo "✅ Services are running:"; \
+		echo "   PolicyEngine API Full: http://localhost:8081"; \
+		echo "   PolicyEngine API Simulation: http://localhost:8082"; \
+		echo "   PolicyEngine API Tagger: http://localhost:8083"; \
+		echo "   Supabase Studio: http://localhost:54323"; \
+		echo "   Supabase API: http://localhost:54321"; \
+		echo "   PostgreSQL: localhost:54322"; \
+	else \
+		echo "⚠️  Supabase CLI not found. Install with: brew install supabase/tap/supabase"; \
+		echo "   PolicyEngine services are still running."; \
+	fi
+	@echo ""
+	@echo "Following logs from PolicyEngine services..."
+	docker-compose -f deployment/docker-compose.yml logs -f
+
+dev-no-supabase:
 	docker-compose -f deployment/docker-compose.yml up --build
 
 up:
@@ -71,7 +95,13 @@ else
 endif
 
 down:
+	@echo "Stopping PolicyEngine services..."
 	docker-compose -f deployment/docker-compose.yml down
+	@echo "Stopping Supabase..."
+	@if command -v supabase >/dev/null 2>&1; then \
+		cd projects/supabase && supabase stop; \
+	fi
+	@echo "✅ All services stopped"
 
 logs:
 ifdef service
@@ -373,3 +403,4 @@ dev-sim:
 
 dev-tagger:
 	docker-compose -f deployment/docker-compose.yml up api-tagger
+

README.md

@@ -9,18 +9,39 @@ Monorepo for PolicyEngine's API infrastructure, containing all services, librari
 - Docker and Docker Compose
 - Python 3.13+
 - [uv](https://docs.astral.sh/uv/) package manager
+- [Supabase CLI](https://supabase.com/docs/guides/cli) (optional, for database features)
 - gcloud CLI (for deployment)
 - Terraform 1.5+ (for deployment)
 
+#### Installing Supabase CLI (macOS)
+```bash
+brew install supabase/tap/supabase
+```
+
 ### Local development
 
-Start all services:
+Start all services including Supabase:
 ```bash
-make up        # Start services on ports 8081-8083
-make logs      # View logs
-make down      # Stop services
+make dev       # Start PolicyEngine services + Supabase (if installed)
+make down      # Stop all services
 ```
 
+Or run PolicyEngine services only:
+```bash
+make dev-no-supabase  # Start only PolicyEngine services
+make up               # Start services detached on ports 8081-8083
+make logs             # View logs
+make down             # Stop services
+```
+
+When running with `make dev`, services are available at:
+- PolicyEngine API Full: http://localhost:8081
+- PolicyEngine API Simulation: http://localhost:8082
+- PolicyEngine API Tagger: http://localhost:8083
+- Supabase API: http://localhost:54321 (if Supabase CLI installed)
+- PostgreSQL: localhost:54322 (if Supabase CLI installed)
+- Email testing (Inbucket): http://localhost:54324 (if Supabase CLI installed)
+
 Run the test suite:
 ```bash
 make test                          # Unit tests only

projects/policyengine-api-full/pyproject.toml

@@ -16,7 +16,10 @@ dependencies = [
     "opentelemetry-instrumentation-sqlalchemy (>=0.51b0,<0.52)",
     "pydantic-settings (>=2.7.1,<3.0.0)",
     "opentelemetry-instrumentation-fastapi (>=0.51b0,<0.52)",
-    "policyengine-fastapi"
+    "policyengine-fastapi",
+    "policyengine[uk,us]>=3.0.0",
+    "psycopg2-binary>=2.9.0",
+    "anthropic>=0.68.1",
 ]
 
 [tool.hatch.build.targets.wheel]

projects/policyengine-api-full/scripts/add_quantile_fields.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+"""
+Add quantile filter fields to aggregates and aggregate_changes tables.
+"""
+
+import os
+from sqlmodel import Session, text
+from policyengine.database import Database
+
+def add_quantile_fields(database_url: str = None):
+    """
+    Add quantile filter fields to existing tables.
+
+    Args:
+        database_url: PostgreSQL connection string. If None, uses DATABASE_URL env var
+    """
+    # Get database URL
+    if database_url is None:
+        database_url = os.environ.get(
+            "DATABASE_URL",
+            "postgresql://postgres:[email protected]:54322/postgres"
+        )
+
+    print(f"Connecting to database: {database_url}")
+    database = Database(database_url)
+
+    with Session(database.engine) as session:
+        # Add quantile fields to aggregates table
+        print("Adding quantile fields to aggregates table...")
+        try:
+            session.exec(text("""
+                ALTER TABLE aggregates
+                ADD COLUMN IF NOT EXISTS filter_variable_quantile_leq DOUBLE PRECISION,
+                ADD COLUMN IF NOT EXISTS filter_variable_quantile_geq DOUBLE PRECISION,
+                ADD COLUMN IF NOT EXISTS filter_variable_quantile_value VARCHAR;
+            """))
+            session.commit()
+            print("✓ Added quantile fields to aggregates table")
+        except Exception as e:
+            print(f"Error adding fields to aggregates: {e}")
+            session.rollback()
+
+        # Add quantile fields to aggregate_changes table
+        print("Adding quantile fields to aggregate_changes table...")
+        try:
+            session.exec(text("""
+                ALTER TABLE aggregate_changes
+                ADD COLUMN IF NOT EXISTS filter_variable_quantile_leq DOUBLE PRECISION,
+                ADD COLUMN IF NOT EXISTS filter_variable_quantile_geq DOUBLE PRECISION,
+                ADD COLUMN IF NOT EXISTS filter_variable_quantile_value VARCHAR;
+            """))
+            session.commit()
+            print("✓ Added quantile fields to aggregate_changes table")
+        except Exception as e:
+            print(f"Error adding fields to aggregate_changes: {e}")
+            session.rollback()
+
+    print("\nMigration complete!")
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Add quantile fields to PolicyEngine database")
+    parser.add_argument(
+        "--database-url",
+        help="PostgreSQL connection string",
+        default="postgresql://postgres:[email protected]:54322/postgres"
+    )
+
+    args = parser.parse_args()
+
+    add_quantile_fields(database_url=args.database_url)

projects/policyengine-api-full/scripts/create_tables.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+"""
+Create database tables for PolicyEngine API.
+"""
+
+import os
+from policyengine.database import Database
+from sqlmodel import SQLModel
+
+def create_tables(database_url: str = None):
+    """
+    Create all database tables.
+
+    Args:
+        database_url: PostgreSQL connection string. If None, uses DATABASE_URL env var
+    """
+    # Get database URL
+    if database_url is None:
+        database_url = os.environ.get(
+            "DATABASE_URL",
+            "postgresql://postgres:[email protected]:54322/postgres"
+        )
+
+    print(f"Connecting to database: {database_url}")
+    database = Database(database_url)
+
+    print("Creating all tables...")
+    # This will create all tables defined in SQLModel
+    SQLModel.metadata.create_all(database.engine)
+
+    print("Tables created successfully!")
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Create PolicyEngine database tables")
+    parser.add_argument(
+        "--database-url",
+        help="PostgreSQL connection string",
+        default="postgresql://postgres:[email protected]:54322/postgres"
+    )
+
+    args = parser.parse_args()
+
+    create_tables(database_url=args.database_url)