diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 4edd7b1a..00000000 --- a/.coveragerc +++ /dev/null @@ -1,2 +0,0 @@ -[run] -relative_files = True diff --git a/.env.example b/.env.example index 047ff4b1..7ee4936e 100644 --- a/.env.example +++ b/.env.example @@ -5,58 +5,48 @@ # ── Application Config ────────────────────────────────────────────── -# Secret key for signing JWT tokens. +# Secret key for signing JWT tokens and Flask sessions. # Generate one: python -c "import secrets; print(secrets.token_urlsafe(32))" -# Required in production +# Required SECRET_KEY=change-me-in-production +# ── Environment & CORS ────────────────────────────── + # Runtime environment. Set to "production" in production. +# In production, ALLOWED_ORIGINS must be set explicitly (CORS will reject all others). # Optional — defaults to "development" ENVIRONMENT=development -# Debug mode. Do NOT enable in production. +# Debug mode. Enables detailed error pages and auto-reload. +# Do NOT enable in production. # Optional — defaults to False # DEBUG=False # Comma-separated list of allowed CORS origins. -# Only used when ENVIRONMENT=production. +# Only used when ENVIRONMENT=production. When empty or during development, all origins are allowed. # Optional — defaults to "http://localhost:3000,http://localhost:7860" ALLOWED_ORIGINS=http://localhost:3000,http://localhost:7860 - # ── Database ───────────────────────────────────────────────── # SQLAlchemy database connection string. # Default: SQLite stored at ./data/app.db -# For Postgres: postgresql://user:pass@host:5432/dbname +# For Postgres: postgresql+asyncpg://user:pass@host:5432/dbname # Optional — defaults to sqlite:///./data/app.db # DATABASE_URL=sqlite:///./data/app.db - -# ── Field-level Encryption ───────────────────────────────── - -# Dedicated key for encrypting sensitive user fields (tokens, secrets). -# Must be set in production — no default value is provided. -# Generate one: python -c "import secrets; print(secrets.token_urlsafe(32))" -# Required in production -# FIELD_ENCRYPTION_KEY=your_32_byte_base64_encoded_key - - # ── Authentication ────────────────────────────────────────── -# JWT signing algorithm. Leave as default (HS256) unless you know what you're doing. +# JWT signing algorithm. Leave as default unless you know what you're doing. # Optional — defaults to "HS256" # JWT_ALGORITHM=HS256 -# JWT access token expiry in minutes. -# Optional — defaults to 15 -# JWT_ACCESS_EXPIRY_MINUTES=15 - -# JWT refresh token expiry in days. -# Optional — defaults to 7 -# JWT_REFRESH_EXPIRY_DAYS=7 +# JWT token expiry in hours. After this period, users must re-login. +# Optional — defaults to 72 +# JWT_EXPIRY_HOURS=72 # Google OAuth client ID for backend ID-token verification. +# Use the same OAuth web client ID as NEXT_PUBLIC_GOOGLE_CLIENT_ID. # Optional — required only for Google sign-in. # GOOGLE_CLIENT_ID=your_google_oauth_client_id.apps.googleusercontent.com @@ -68,6 +58,11 @@ ALLOWED_ORIGINS=http://localhost:3000,http://localhost:7860 # Optional — required only for Google Drive sync. # GOOGLE_DRIVE_REDIRECT_URI=http://localhost:8000/api/v1/auth/google-drive/callback +# Google OAuth client ID exposed to the Next.js frontend for Google Identity Services. +# Add the frontend origin to your OAuth web client's authorized JavaScript origins. +# Optional — required only for Google sign-in. +# NEXT_PUBLIC_GOOGLE_CLIENT_ID=your_google_oauth_client_id.apps.googleusercontent.com + # Public frontend URL used to build email verification links. # Optional — defaults to http://localhost:3000 FRONTEND_URL=http://localhost:3000 @@ -76,10 +71,13 @@ FRONTEND_URL=http://localhost:3000 # Optional — defaults to 24 # EMAIL_VERIFICATION_TOKEN_EXPIRE_HOURS=24 - -# ── SMTP / Email ─────────────────────────────────────────── - # SMTP settings used to send account verification emails. +# Required for password registrations to receive real email verification links. +# Gmail setup: +# 1. Enable 2-Step Verification on the sender Google account. +# 2. Create an App Password from Google Account > Security > App passwords. +# 3. Use the Gmail address as MAIL_USERNAME and MAIL_FROM. +# 4. Use the 16-character App Password as MAIL_PASSWORD. MAIL_USERNAME=your_smtp_username MAIL_PASSWORD=your_smtp_or_gmail_app_password MAIL_FROM=your_sender_email@example.com @@ -88,6 +86,14 @@ MAIL_PORT=587 MAIL_STARTTLS=True MAIL_SSL_TLS=False +# Gmail example: +# MAIL_USERNAME=yourgmail@gmail.com +# MAIL_PASSWORD=your_16_character_app_password +# MAIL_FROM=yourgmail@gmail.com +# MAIL_SERVER=smtp.gmail.com +# MAIL_PORT=587 +# MAIL_STARTTLS=True +# MAIL_SSL_TLS=False # ── Celery / Redis Background Processing ─────────────────── @@ -99,22 +105,20 @@ MAIL_SSL_TLS=False # Optional — defaults to redis://localhost:6379/1 # CELERY_RESULT_BACKEND=redis://localhost:6379/1 - # ── File Upload ───────────────────────────────────────────── -# Directory where uploaded documents are stored. +# Directory where uploaded documents (PDFs, DOCXs, etc.) are stored. # Optional — defaults to "./data/uploads" # UPLOAD_DIR=./data/uploads # Maximum upload file size in megabytes. # Optional — defaults to 50 -# MAX_UPLOAD_SIZE_MB=50 +# MAX_FILE_SIZE_MB=50 -# Comma-separated list of allowed file extensions. +# Comma-separated list of allowed file extensions for upload. # Optional — defaults to "pdf,docx,txt,md" # ALLOWED_EXTENSIONS=pdf,docx,txt,md - # ── HuggingFace (Required for LLM inference and OAuth) ─────── # HuggingFace API token. Used to call the Inference API for LLM responses. @@ -122,18 +126,19 @@ MAIL_SSL_TLS=False # Required (app won't generate answers without it) HF_TOKEN=your_huggingface_token_here -# HuggingFace OAuth client ID and secret for native login support +# HuggingFace OAuth variables for native login support # Optional — required only for Hugging Face sign-in -# HF_CLIENT_ID=your_hf_oauth_client_id -# HF_CLIENT_SECRET=your_hf_oauth_client_secret -# HF_REDIRECT_URI=http://localhost:8000/api/v1/auth/callback/huggingface - +HF_CLIENT_ID=your_hf_oauth_client_id +HF_CLIENT_SECRET=your_hf_oauth_client_secret +HF_REDIRECT_URI=http://localhost:8000/api/v1/auth/callback/huggingface +FRONTEND_URL=http://localhost:3000 # ── LLM Configuration ─────────────────────────────────────── # HuggingFace model ID used for answer generation. -# Optional — defaults to "Qwen/Qwen2.5-72B-Instruct" -# LLM_MODEL=Qwen/Qwen2.5-72B-Instruct +# Check available models: https://huggingface.co/models?inference=warm&sort=trending +# Optional — defaults to "mistralai/Mistral-7B-Instruct-v0.3" +# LLM_MODEL=mistralai/Mistral-7B-Instruct-v0.3 # Sampling temperature (0.0 = deterministic, 1.0 = very creative). # Optional — defaults to 0.3 @@ -143,7 +148,6 @@ HF_TOKEN=your_huggingface_token_here # Optional — defaults to 1024 # LLM_MAX_NEW_TOKENS=1024 - # ── LangSmith Tracing (Optional) ──────────────────────── # Enable LangSmith tracing for the backend RAG pipeline. @@ -162,8 +166,7 @@ HF_TOKEN=your_huggingface_token_here # Optional — defaults to "pdf-assistant-rag" # LANGSMITH_PROJECT=pdf-assistant-rag - -# ── Embeddings ────────────────────────────────────────────── +# ── Embeddings (Optional — defaults shown)────────────────────────────────────────────── # SentenceTransformer model ID for generating document embeddings. # Model is downloaded once and cached locally. No external API call. @@ -174,31 +177,9 @@ HF_TOKEN=your_huggingface_token_here # Optional — defaults to 384 # EMBEDDING_DIMENSION=384 +# ── RAG Config (Optional — defaults shown) ─────────── -# ── RAG Config ────────────────────────────────────────────── - -# Number of characters per document chunk. -# Optional — defaults to 1000 -# CHUNK_SIZE=1000 - -# Character overlap between consecutive chunks. -# Optional — defaults to 200 -# CHUNK_OVERLAP=200 - -# Number of candidate chunks retrieved during semantic search. -# Optional — defaults to 20 -# TOP_K_RETRIEVAL=20 - -# Number of top chunks passed to the LLM after reranking. -# Optional — defaults to 8 -# TOP_K_RERANK=8 - -# Cross-encoder model used for reranking. -# Optional — defaults to "BAAI/bge-reranker-v2-m3" -# RERANKER_MODEL=BAAI/bge-reranker-v2-m3 - - -# ── Knowledge Graph / GraphRAG ────────────────────────────── +# ── Knowledge Graph / GraphRAG (Optional — defaults shown) ───────────────── # Directory where GraphRAG stores per-document knowledge graphs. # Optional — defaults to "./data/graphs" @@ -208,47 +189,41 @@ HF_TOKEN=your_huggingface_token_here # Optional — defaults to 12 # GRAPH_MAX_RELATIONSHIPS=12 - # ── ChromaDB (Vector Store) ───────────────────────────────── # Directory where ChromaDB persists its vector index to disk. # Optional — defaults to "./data/chroma_db" # CHROMA_PERSIST_DIR=./data/chroma_db +# ── Document Chunking ─────────────────────────────────────── -# ── Document Cleanup ──────────────────────────────────────── - -# Enable automatic cleanup of inactive active documents. -# Optional — defaults to True -# DOC_CLEANUP_ENABLED=True - -# Number of days without access before an active document is purged. -# Optional — defaults to 30 -# DOC_CLEANUP_INACTIVE_DAYS=30 - -# Number of days a soft-deleted document is kept before permanent deletion. -# Optional — defaults to 90 -# DOC_CLEANUP_MAX_AGE_DAYS=90 - - -# ── Workspace Invitations ────────────────────────────────── +# Number of characters per document chunk. +# Larger chunks give more context; smaller chunks improve retrieval precision. +# Optional — defaults to 1000 +# CHUNK_SIZE=1000 -# Public-facing app URL used in invitation emails. -# Optional — defaults to "http://localhost:3000" -# APP_URL=http://localhost:3000 +# Character overlap between consecutive chunks. Helps maintain context at boundaries. +# Optional — defaults to 200 +# CHUNK_OVERLAP=200 -# Invitation token expiry in hours. -# Optional — defaults to 72 -# INVITE_TOKEN_EXPIRY_HOURS=72 +# ── Retrieval ─────────────────────────────────────────────── +# Number of candidate chunks retrieved from the vector store during semantic search. +# Optional — defaults to 10 +# TOP_K_RETRIEVAL=10 -# ── Response Caching ─────────────────────────────────────── +# Number of top chunks passed to the LLM after cross-encoder reranking. +# Must be ≤ TOP_K_RETRIEVAL. +# Optional — defaults to 5 +# TOP_K_RERANK=5 -# Redis connection URL. Leave empty to use in-memory LRU fallback. -# REDIS_URL= +# Cross-encoder model used for reranking retrieved chunks by relevance. +# Optional — defaults to "cross-encoder/ms-marco-MiniLM-L-6-v2" +# RERANKER_MODEL=cross-encoder/ms-marco-MiniLM-L-6-v2 -# Cache TTL in seconds (default: 3600 = 1 hour) -# CACHE_TTL=3600 +# ── (Legacy) Flask-Only Variables ─────────────────────────── +# These are only used if you run the old Flask app (app.py) instead of FastAPI. +# They are ignored by the new FastAPI backend. -# Max entries for in-memory LRU cache when Redis is unavailable -# CACHE_LRU_MAX_SIZE=128 +# MONGO_URI=mongodb://localhost:27017/pdf_assistant +# GOOGLE_CLIENT_SECRET=your_google_client_secret diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..1d47bee8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,255 @@ +name: CI — Dev Branch + +# ────────────────────────────────────────────────────────── +# Only runs on: +# • Pushes to dev +# • PRs targeting dev +# ────────────────────────────────────────────────────────── +on: + push: + branches: ["dev"] + pull_request: + branches: ["dev"] + +jobs: + # ── 1. Backend Lint & Import Check ─────────────────────_ + backend-check: + name: 🐍 Backend — Lint & Import Check + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Cache Pip packages + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('backend/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install flake8 flake8-bugbear + # Install project deps (skip heavy ML libs with stub extras) + pip install -r backend/requirements.txt --quiet || true + + - name: Flake8 lint (errors only, no style noise) + run: | + flake8 backend/app \ + --max-line-length=120 \ + --select=E9,F63,F7,F82 \ + --count \ + --show-source \ + --statistics + + - name: Check all Python modules import cleanly + env: + SECRET_KEY: ci-dummy-secret + DATABASE_URL: sqlite:///./ci_test.db + DEBUG: "false" + HF_TOKEN: ci-dummy-token + UPLOAD_DIR: /tmp/uploads + CHROMA_PERSIST_DIR: /tmp/chroma + run: | + python -c "import sys; sys.path.insert(0, 'backend'); from app.config import get_settings; get_settings(); print('Config imports OK')" + + - name: Install pytest-cov + run: pip install pytest-cov + + - name: Run backend pytest suite with coverage + env: + SECRET_KEY: ci-dummy-secret + DATABASE_URL: sqlite:///./ci_test.db + DEBUG: "false" + HF_TOKEN: ci-dummy-token + UPLOAD_DIR: /tmp/uploads + CHROMA_PERSIST_DIR: /tmp/chroma + run: | + pytest backend/tests -v \ + --cov=backend/app \ + --cov-report=term-missing \ + --cov-report=xml:coverage.xml \ + --cov-report=html:htmlcov \ + --cov-fail-under=40 + + - name: Upload coverage XML report + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: | + coverage.xml + htmlcov/ + retention-days: 7 + + - name: Coverage summary comment (PR only) + if: github.event_name == 'pull_request' + uses: py-cov-action/python-coverage-comment-action@v3 + with: + GITHUB_TOKEN: ${{ github.token }} + MINIMUM_GREEN: 40 + MINIMUM_ORANGE: 30 + continue-on-error: true + + # ── 2. CodeQL Static Security Analysis ────────────────── + codeql-analysis: + name: 🔎 CodeQL — Static Security Analysis (${{ matrix.language }}) + runs-on: ubuntu-latest + + permissions: + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + language: ["python", "javascript-typescript"] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: ${{ matrix.language }} + queries: +security-extended,security-and-quality + + - name: Perform CodeQL analysis + uses: github/codeql-action/analyze@v4 + with: + category: "/language:${{ matrix.language }}" + output: ${{ runner.temp }}/codeql-results/${{ matrix.language }} + upload: false + + - name: Fail on critical security findings + env: + SARIF_DIR: ${{ runner.temp }}/codeql-results/${{ matrix.language }} + run: | + python - <<'PY' + import json + import os + import pathlib + import sys + + sarif_dir = pathlib.Path(os.environ["SARIF_DIR"]) + critical_findings = [] + + for sarif_path in sarif_dir.rglob("*.sarif"): + with sarif_path.open(encoding="utf-8") as handle: + sarif = json.load(handle) + + for run in sarif.get("runs", []): + rule_severity = { + rule.get("id"): float( + rule.get("properties", {}).get( + "security-severity", + "0", + ) + ) + for rule in run.get("tool", {}) + .get("driver", {}) + .get("rules", []) + if rule.get("id") + } + + for result in run.get("results", []): + rule_id = result.get("ruleId") + severity = rule_severity.get(rule_id, 0.0) + if severity < 9.0: + continue + + location = result.get("locations", [{}])[0].get( + "physicalLocation", + {}, + ) + artifact = location.get("artifactLocation", {}).get( + "uri", + "unknown file", + ) + region = location.get("region", {}) + line = region.get("startLine", "?") + message = result.get("message", {}).get("text", "") + critical_findings.append( + f"{rule_id} ({severity}) at {artifact}:{line} — {message}" + ) + + if critical_findings: + print("Critical CodeQL security findings detected:") + for finding in critical_findings: + print(f"- {finding}") + sys.exit(1) + + print("No critical CodeQL security findings detected.") + PY + + # ── 3. Frontend Build Check ───────────────────────────── + frontend-check: + name: ⚛️ Frontend — TypeScript & Build + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Cache Node modules + uses: actions/cache@v4 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('frontend/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + + - name: Set up Node.js 20 + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install dependencies + working-directory: frontend + run: npm ci + + - name: TypeScript type-check + working-directory: frontend + run: npx tsc --noEmit + + - name: ESLint + working-directory: frontend + run: npm run lint + + - name: Next.js build (production bundle check) + working-directory: frontend + run: npm run build + env: + NEXT_PUBLIC_API_URL: http://localhost:8000 + + # ── 4. PR Size Gate ───────────────────────────────────── + pr-size-check: + name: 📏 PR Size Check + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Count changed lines + run: | + LINES=$(git diff --stat origin/${{ github.base_ref }}...HEAD | tail -1) + echo "Changes: $LINES" + INSERTIONS=$(git diff --numstat origin/${{ github.base_ref }}...HEAD | awk '{sum+=$1} END{print sum}') + if [ "$INSERTIONS" -gt 1000 ]; then + echo "⚠️ PR is very large ($INSERTIONS lines added). Please consider splitting it." + else + echo "✅ PR size looks good ($INSERTIONS lines added)." + fi diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 00000000..37be49d2 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,35 @@ +name: Frontend E2E + +on: + pull_request: + branches: ["dev"] + push: + branches: ["dev"] + +jobs: + playwright: + name: Playwright + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + cache-dependency-path: frontend/package-lock.json + + - name: Install frontend dependencies + working-directory: frontend + run: npm ci + + - name: Install Playwright browser + working-directory: frontend + run: npx playwright install --with-deps chromium + + - name: Run Playwright E2E tests + working-directory: frontend + run: npm run test:e2e -- --reporter=list diff --git a/.github/workflows/sync-issue-labels.yml b/.github/workflows/sync-issue-labels.yml new file mode 100644 index 00000000..be7fa97b --- /dev/null +++ b/.github/workflows/sync-issue-labels.yml @@ -0,0 +1,164 @@ +name: Sync Labels — Issue to PR + +# ────────────────────────────────────────────────────────── +# Auto-syncs labels from referenced issue(s) to the PR when +# a PR is opened or updated targeting `dev`. +# +# Why pull_request_target: +# Label operations need write permissions on the target +# repo. pull_request_target runs in the context of the +# base repo with access to secrets and write token. +# Since we only read issue data and apply labels, there +# is no security concern. +# ────────────────────────────────────────────────────────── + +on: + pull_request_target: + types: [closed] + branches: ["dev"] + +permissions: + contents: read + issues: read + pull-requests: write + +jobs: + sync-labels: + name: Sync labels from referenced issue + runs-on: ubuntu-latest + if: github.event.pull_request.merged == true + + steps: + - name: Extract issue numbers from PR body + id: extract + env: + PR_BODY: ${{ github.event.pull_request.body }} + run: | + # Match patterns: + # "Closes #123" + # "Fixes #456, #789" (comma-separated) + # "Resolves #111, #222, #333" + # + # Approach: grab lines containing a keyword, then + # extract every NNN from those lines. + # We place '|| true' at the very end of the pipeline so it doesn't short-circuit. + ISSUES=$( + echo "${PR_BODY:-}" \ + | grep -ioE '.*(closes|fixes|resolves).*' \ + | grep -oE '#[0-9]+' \ + | grep -oE '[0-9]+' \ + | sort -un \ + | xargs \ + || true + ) + echo "Found issues: [$ISSUES]" + echo "issues=$ISSUES" >> "$GITHUB_OUTPUT" + + - name: Fetch and apply labels + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ISSUES: ${{ steps.extract.outputs.issues }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + + ALL_LABELS="gssoc"$'\n'"gssoc:approved"$'\n'"mentor:param20h"$'\n' + + for ISSUE in $ISSUES; do + echo "--- Fetching labels for #$ISSUE ---" + + LABELS=$(gh issue view "$ISSUE" --repo "$REPO" --json labels --jq '.labels[].name' 2>/dev/null || true) + + if [ -z "$LABELS" ]; then + echo " → No labels on #$ISSUE, skipping" + continue + fi + + echo " → Labels: $(echo "$LABELS" | tr '\n' ' ')" + + # Accumulate labels (newline-separated, deduplicated later) + ALL_LABELS="${ALL_LABELS}${LABELS}"$'\n' + done + + if [ -z "$ALL_LABELS" ]; then + echo "No labels to apply. Exiting." + exit 0 + fi + + # Deduplicate and remove empty lines + UNIQUE_LABELS=$(echo "$ALL_LABELS" | sort -u | grep -v '^$') + + echo "" + echo "=== Applying labels to PR #$PR_NUMBER ===" + echo "$UNIQUE_LABELS" + + # Get labels already on the PR + EXISTING=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json labels --jq '.labels[].name' 2>/dev/null || true) + + MISSING=0 + while IFS= read -r LABEL; do + [ -z "$LABEL" ] && continue + if echo "$EXISTING" | grep -qxF "$LABEL"; then + echo " ✓ Already present: $LABEL" + else + echo " + Adding: $LABEL" + gh label create "$LABEL" --repo "$REPO" 2>/dev/null || true # create if not exists + gh pr edit "$PR_NUMBER" --repo "$REPO" --add-label "$LABEL" + MISSING=$((MISSING + 1)) + fi + done <<< "$UNIQUE_LABELS" + + if [ "$MISSING" -eq 0 ]; then + echo "All labels already synced — nothing to add." + else + echo "Done. Added $MISSING label(s) to PR #$PR_NUMBER." + fi + + - name: Calculate GSSoC Points and Comment + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + + echo "Calculating GSSoC points for PR #$PR_NUMBER..." + + # Fetch all labels currently on the PR (including the ones we just synced) + PR_LABELS=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json labels --jq '.labels[].name' 2>/dev/null || true) + + POINTS=0 + + while IFS= read -r LABEL; do + [ -z "$LABEL" ] && continue + case "$LABEL" in + "level:beginner") POINTS=$((POINTS + 20)) ;; + "level:intermediate") POINTS=$((POINTS + 35)) ;; + "level:advanced") POINTS=$((POINTS + 55)) ;; + "level:critical") POINTS=$((POINTS + 80)) ;; + "type:accessibility") POINTS=$((POINTS + 15)) ;; + "type:bug") POINTS=$((POINTS + 10)) ;; + "type:design") POINTS=$((POINTS + 10)) ;; + "type:devops") POINTS=$((POINTS + 15)) ;; + "type:docs") POINTS=$((POINTS + 5)) ;; + "type:feature") POINTS=$((POINTS + 10)) ;; + "type:performance") POINTS=$((POINTS + 15)) ;; + "type:refactor") POINTS=$((POINTS + 10)) ;; + "type:security") POINTS=$((POINTS + 20)) ;; + "type:testing") POINTS=$((POINTS + 10)) ;; + esac + done <<< "$PR_LABELS" + + echo "Total Points Calculated: $POINTS" + + if [ "$POINTS" -gt 0 ]; then + printf -v COMMENT "🎉 **Congratulations on getting your Pull Request merged!** 🎉\n\nThank you for contributing to PDF-Assistant-RAG as part of GSSoC '26! 🚀\n\nKeep up the great work! ✨" + + # Post the comment to the PR + echo "Posting comment..." + gh pr comment "$PR_NUMBER" --repo "$REPO" --body "$COMMENT" + else + echo "No scorable labels found. No comment posted." + fi + diff --git a/.secrets.baseline b/.secrets.baseline deleted file mode 100644 index 76a4845c..00000000 --- a/.secrets.baseline +++ /dev/null @@ -1,196 +0,0 @@ -{ - "version": "1.5.0", - "plugins_used": [ - { - "name": "ArtifactoryDetector" - }, - { - "name": "AWSKeyDetector" - }, - { - "name": "AzureStorageKeyDetector" - }, - { - "name": "Base64HighEntropyString", - "limit": 4.5 - }, - { - "name": "BasicAuthDetector" - }, - { - "name": "CloudantDetector" - }, - { - "name": "DiscordBotTokenDetector" - }, - { - "name": "GitHubTokenDetector" - }, - { - "name": "GitLabTokenDetector" - }, - { - "name": "HexHighEntropyString", - "limit": 3.0 - }, - { - "name": "IbmCloudIamDetector" - }, - { - "name": "IbmCosHmacDetector" - }, - { - "name": "IPPublicDetector" - }, - { - "name": "JwtTokenDetector" - }, - { - "name": "KeywordDetector", - "keyword_exclude": "" - }, - { - "name": "MailchimpDetector" - }, - { - "name": "NpmDetector" - }, - { - "name": "OpenAIDetector" - }, - { - "name": "PrivateKeyDetector" - }, - { - "name": "PypiTokenDetector" - }, - { - "name": "SendGridDetector" - }, - { - "name": "SlackDetector" - }, - { - "name": "SoftlayerDetector" - }, - { - "name": "SquareOAuthDetector" - }, - { - "name": "StripeDetector" - }, - { - "name": "TelegramBotTokenDetector" - }, - { - "name": "TwilioKeyDetector" - } - ], - "filters_used": [ - { - "path": "detect_secrets.filters.allowlist.is_line_allowlisted" - }, - { - "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", - "min_level": 2 - }, - { - "path": "detect_secrets.filters.heuristic.is_indirect_reference" - }, - { - "path": "detect_secrets.filters.heuristic.is_likely_id_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_lock_file" - }, - { - "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_potential_uuid" - }, - { - "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" - }, - { - "path": "detect_secrets.filters.heuristic.is_sequential_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_swagger_file" - }, - { - "path": "detect_secrets.filters.heuristic.is_templated_secret" - } - ], - "results": { - ".env.example": [ - { - "type": "Basic Auth Credentials", - "filename": ".env.example", - "hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684", - "is_verified": false, - "line_number": 34 - } - ], - "backend\\tests\\conftest.py": [ - { - "type": "Secret Keyword", - "filename": "backend\\tests\\conftest.py", - "hashed_secret": "784ea9498a644d21ccac29eb7f5a078b866013ba", - "is_verified": false, - "line_number": 19 - } - ], - "backend\\tests\\test_auth.py": [ - { - "type": "Secret Keyword", - "filename": "backend\\tests\\test_auth.py", - "hashed_secret": "cbfdac6008f9cab4083784cbd1874f76618d2a97", - "is_verified": false, - "line_number": 7 - }, - { - "type": "Secret Keyword", - "filename": "backend\\tests\\test_auth.py", - "hashed_secret": "6809ffccad03b80fa1fbc32c17e7e054805ec30b", - "is_verified": false, - "line_number": 58 - }, - { - "type": "Secret Keyword", - "filename": "backend\\tests\\test_auth.py", - "hashed_secret": "03f5e2d670af3e9183f3fe790785b0d41291a17d", - "is_verified": false, - "line_number": 150 - } - ], - "backend\\tests\\test_document_upload_validation.py": [ - { - "type": "Secret Keyword", - "filename": "backend\\tests\\test_document_upload_validation.py", - "hashed_secret": "6318553899daae2941718c02508aeee938af1a1c", - "is_verified": false, - "line_number": 120 - } - ], - "bots\\discord\\README.md": [ - { - "type": "Secret Keyword", - "filename": "bots\\discord\\README.md", - "hashed_secret": "ebf268ab0b5c6ac1d7a23ae864683a15f60b7a44", - "is_verified": false, - "line_number": 22 - } - ], - "docker-compose.yml": [ - { - "type": "Secret Keyword", - "filename": "docker-compose.yml", - "hashed_secret": "d033e22ae348aeb5660fc2140aec35850c4da997", - "is_verified": false, - "line_number": 102 - } - ] - }, - "generated_at": "2026-06-04T18:14:24Z" -} diff --git a/CHANGELOG.MD b/CHANGELOG.MD new file mode 100644 index 00000000..969f63a4 --- /dev/null +++ b/CHANGELOG.MD @@ -0,0 +1,54 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.4.0] - 2026-05-16 +### Added +- Configured contributor workflow on the `dev` branch. + +### Fixed +- Resolved React Hook linting errors (`react-hooks/set-state-in-effect`) in CI pipelines by lazy-initializing the loading state to prevent `setLoading(false)` execution inside the effect body. +- Fixed chat scroll component tracking using a `bottomRef` sentinel and `scrollIntoView` mechanism to replace the broken `scrollRef` on the ScrollArea wrapper. + +### Documentation +- Extensively overhauled `README.md` to add full project documentation, an explicit RAG pipeline architectural diagram, comprehensive API reference, and contribution guides. + +## [0.3.0] - 2026-04-15 +### Added +- Implemented a brand new UI and upgraded internal RAG model architectures. +- Configured native Hugging Face Spaces Docker deployment handling non-root user execution, model pre-download caching, and custom keep-alive timeouts. + +### Changed +- Switched default open-source inference engine to `Qwen2.5-72B-Instruct` to leverage Hugging Face free-tier hardware. + +### Fixed +- Patched critical `list index out of range` runtime crash by explicitly handling empty choice selections from LLM responses. +- Adjusted production API routing to enforce same-origin API calls and added native `HEAD` method support to satisfy Next.js route prefetching rules. +- Upgraded text chunking modules to use `langchain_text_splitters` to ensure compatibility with LangChain v0.3+. +- Removed bulky compiled binary assets before pushing to Hugging Face Spaces storage layers. + +## [0.2.0] - 2026-02-26 +### Added +- Implemented an alternative lightweight RAG pipeline utilizing a Pinecone vector database index, Gemini embeddings, and Render hosting deployment profiles. +- Integrated automated Google Cloud Run continuous deployment (CD) workflows via Google Cloud Platform (GCP). +- Added `ProxyFix` middleware support to securely preserve OAuth authentication headers behind Render's reverse proxy structure. + +### Fixed +- Cleaned hardcoded testing credentials and placeholder URIs flagged during automated GitHub secret scanning routines. + +## [0.1.0] - 2024-06-25 +### Added +- Initialized core repository, licensing infrastructure, and baseline documentation assets. +- Built initial RAG application architecture featuring file ingestion systems parsing raw `.txt`, `.docx`, and `.md` formats. +- Implemented native Google Authentication security layers. + +[unreleased]: https://github.com/param20h/PDF-Assistant-RAG/compare/v0.4.0...HEAD +[0.4.0]: https://github.com/param20h/PDF-Assistant-RAG/compare/v0.3.0...v0.4.0 +[0.3.0]: https://github.com/param20h/PDF-Assistant-RAG/compare/v0.2.0...v0.3.0 +[0.2.0]: https://github.com/param20h/PDF-Assistant-RAG/compare/v0.1.0...v0.2.0 +[0.1.0]: https://github.com/param20h/PDF-Assistant-RAG/commits/dev \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5fe7cd30..c733c7fb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -206,3 +206,7 @@ All checks must be green before your PR can be merged. ## 💬 Need Help? Open a [Discussion](https://github.com/param20h/PDF-Assistant-RAG/discussions) before opening an issue if you're unsure. Mentors and the admin check discussions regularly. + +--- + +Thanks for contributing! Every PR, no matter how small, makes a difference. 🚀 diff --git a/Dockerfile b/Dockerfile index 1e177c6b..fbd9ce50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,22 @@ # syntax=docker/dockerfile:1 # -------------------------------------------------------- -# Stage 1: Build Python dependencies in an isolated venv +# Stage 1: Build Next.js frontend assets +# -------------------------------------------------------- +FROM node:20-alpine AS frontend-builder + +WORKDIR /app/frontend + +# Install dependencies +COPY frontend/package.json frontend/package-lock.json ./ +RUN npm ci --no-audit + +# Copy frontend source and build +COPY frontend/ ./ +RUN npm run build + +# -------------------------------------------------------- +# Stage 2: Build Python dependencies in an isolated venv # -------------------------------------------------------- FROM python:3.11-slim AS python-builder @@ -25,7 +40,7 @@ RUN pip install --no-cache-dir --upgrade pip setuptools wheel && \ find /opt/venv -type f -name "*.pyc" -delete # -------------------------------------------------------- -# Stage 2: Runtime image with only backend code and dependencies +# Stage 3: Runtime image with only app code and artifacts # -------------------------------------------------------- FROM python:3.11-slim @@ -40,7 +55,7 @@ RUN useradd -m -u 1000 appuser WORKDIR /app -# Runtime-only system packages +# Runtime-only system packages. Build tools stay in python-builder. RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ libmagic1 \ @@ -53,6 +68,9 @@ COPY --from=python-builder /opt/venv /opt/venv COPY backend/app ./app COPY backend/__init__.py ./backend/__init__.py +# Copy frontend build from stage 1 +COPY --from=frontend-builder /app/frontend/out ./frontend/out + # Create data directories with proper permissions RUN mkdir -p /app/data/uploads /app/data/chroma_db /app/data/graphs /app/data/huggingface && \ chown -R appuser:appuser /app @@ -64,3 +82,4 @@ USER appuser EXPOSE 7860 CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"] + diff --git a/README.md b/README.md index 6dee67fd..9bb9a743 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ short_description: Enterprise Agentic RAG — upload PDFs and chat with AI ██╔═══╝ ██║ ██║██╔══╝ ██╔══██║╚════██║╚════██║██║╚════██║ ██║ ██╔══██║██║╚██╗██║ ██║ ██║ ██████╔╝██║ ██║ ██║███████║███████║██║███████║ ██║ ██║ ██║██║ ╚████║ ██║ ╚═╝ ╚═════╝ ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚══════╝ ╚═╝ ╚═╝ ╚═╝╚═╝ ╚═══╝ ╚═╝ - + ██████╗ █████╗ ██████╗ ██╔══██╗██╔══██╗██╔════╝ ██████╔╝███████║██║ ███╗ @@ -34,13 +34,12 @@ short_description: Enterprise Agentic RAG — upload PDFs and chat with AI
-[![FastAPI](https://img.shields.io/badge/FastAPI-0.115+-009688?style=for-the-badge&logo=fastapi&logoColor=white)](https://fastapi.tiangolo.com/) -[![Next.js](https://img.shields.io/badge/Next.js-16-000000?style=for-the-badge&logo=nextdotjs&logoColor=white)](https://nextjs.org/) +[![Flask](https://img.shields.io/badge/Flask-2.x-000000?style=for-the-badge&logo=flask&logoColor=white)](https://flask.palletsprojects.com/) [![Python](https://img.shields.io/badge/Python-3.11-3776AB?style=for-the-badge&logo=python&logoColor=white)](https://python.org/) -[![PostgreSQL](https://img.shields.io/badge/PostgreSQL-16-4169E1?style=for-the-badge&logo=postgresql&logoColor=white)](https://postgresql.org/) -[![ChromaDB](https://img.shields.io/badge/ChromaDB-VectorDB-FF6B35?style=for-the-badge)](https://trychroma.com/) -[![HuggingFace](https://img.shields.io/badge/HuggingFace-Inference-FFD21E?style=for-the-badge&logo=huggingface&logoColor=black)](https://huggingface.co/) -[![Celery](https://img.shields.io/badge/Celery-Redis-37814A?style=for-the-badge&logo=celery&logoColor=white)](https://docs.celeryq.dev/) +[![MongoDB](https://img.shields.io/badge/MongoDB-Atlas-47A248?style=for-the-badge&logo=mongodb&logoColor=white)](https://mongodb.com/) +[![Pinecone](https://img.shields.io/badge/Pinecone-VectorDB-000000?style=for-the-badge)](https://pinecone.io/) +[![Groq](https://img.shields.io/badge/Groq-LLM-F55036?style=for-the-badge)](https://console.groq.com/) +[![Gemini](https://img.shields.io/badge/Google_Gemini-Embeddings-4285F4?style=for-the-badge&logo=google&logoColor=white)](https://aistudio.google.com/) [![Docker](https://img.shields.io/badge/Docker-Multi--Stage-2496ED?style=for-the-badge&logo=docker&logoColor=white)](https://docker.com/) [![License: MIT](https://img.shields.io/badge/License-MIT-F59E0B?style=for-the-badge)](LICENSE) @@ -48,11 +47,6 @@ short_description: Enterprise Agentic RAG — upload PDFs and chat with AI > **Upload · Embed · Retrieve · Chat** — A production-grade AI document assistant built end-to-end with an agentic RAG pipeline, streaming responses, and per-user data isolation. -
-## 🌟 GirlScript Summer of Code 2026 - -This project is an official participant in **GirlScript Summer of Code 2026 (GSSoC'26)** and welcomes contributions from the community. -
[Features](#-key-features) · [Tech Stack](#-tech-stack) · [Getting Started](#-getting-started) · [Architecture](#-architecture) · [RAG Pipeline](#-rag-pipeline) · [API Reference](#-api-reference) · [Deployment](#-deployment) · [Contributing](#-contributing) @@ -65,14 +59,31 @@ This project is an official participant in **GirlScript Summer of Code 2026 (GSS Thanks to all the amazing people who have contributed to **PDF-Assistant-RAG**! 🎉 + +### 📋 Contributions +
- - - + +| Avatar | Contributor | Role | Key Contributions | +|:------:|:-----------:|:----:|:-----------------| +| | [**@param20h**](https://github.com/param20h) — Paramjit Singh | 🧭 **Project Lead** | Founded the project; core RAG architecture (FastAPI + ChromaDB + Next.js); Docker multi-stage & HuggingFace Spaces deployment; GitHub Actions CI/CD; JWT auth & Google OAuth; documentation with pipeline diagrams; project governance | +| | [**@Yuvraj-Sarathe**](https://github.com/Yuvraj-Sarathe) — Yuvraj Sarathe | 📐 **Docs & Build** | Added Mermaid architecture diagram; inline RAG pipeline comments; `.env.example` docs; created `Makefile` with concurrent dev commands & `CHANGELOG.md` | +| | [**@SatyamPrakash09**](https://github.com/SatyamPrakash09) — Satyam Prakash | ⚙️ **Backend Engineer** | Chat history export & auto-refresh auth; user profile endpoints; `/health` endpoint (Vector DB + SQL DB monitoring); document pagination; MIME file validation; JWT access + refresh token system | +| | [**@akmhatey-ai**](https://github.com/akmhatey-ai) | 🎨 **UI/UX** | Chat textarea auto-resize; clear messages on document switch | +| | [**@drishtisharma14052007-eng**](https://github.com/drishtisharma14052007-eng) — Drishti Sharma | 📝 **Documentation** | GSSOC contributor FAQ content | +| | [**@Pika-pika06**](https://github.com/Pika-pika06) — Pika | 📝 **Documentation** | Changelog tracking historical commits through v0.4.0 | +| | [**@blinkerbit**](https://github.com/blinkerbit) / [@algojogacor](https://github.com/algojogacor) — Arya Rizky | 🐛 **Bug Buster** | Fixed 0-indexed page number display in source cards | +| | [**@HirenGajjar**](https://github.com/HirenGajjar) | 🔒 **DevOps** | Restricted CORS origins via `ALLOWED_ORIGINS` environment variable | +| | [**@Kaustub26Pvgda**](https://github.com/Kaustub26Pvgda) — Kaustub Pavagada | ⚡ **Frontend Engineer** | Copy LLM response capability; increased max file upload size to 20 MB | +| | [**@akshy-yy**](https://github.com/akshy-yy) — Akshaya | 🎨 **Frontend Engineer** | Typing indicator animation while AI responds | +| | [**@GHX5T-SOL**](https://github.com/GHX5T-SOL) — Bruce Wayne | 🐛 **Bug Buster** | Backend offline error message display | +| | [**@viswanatha**](https://github.com/viswanatha) | 📊 **Observability** | Health check endpoint | +
+
-> 🌟 **Want to join them?** Check out [CONTRIBUTING.md](CONTRIBUTING.md) for contribution guidelines and look for [good first issues](https://github.com/param20h/PDF-Assistant-RAG/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) to get started! +> 🌟 **Want to join them?** Check out [CONTRIBUTING.md](CONTRIBUTING.md) for contribution guidelines and look for [good first issues](https://github.com/Yuvraj-Sarathe/PDF-Assistant-RAG/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) to get started! --- @@ -82,138 +93,82 @@ Thanks to all the amazing people who have contributed to **PDF-Assistant-RAG**! **PDF-Assistant-RAG** is a complete, production-ready AI document assistant that lets users upload complex PDFs, financial reports, legal contracts, and research papers — then chat with an AI that provides **accurate, cited answers** powered by a multi-stage Retrieval-Augmented Generation pipeline. -The system uses **hybrid search (vector + BM25) with Reciprocal Rank Fusion** and **cross-encoder reranking** to find the most relevant document chunks, streams AI-generated answers token-by-token, and highlights exact source citations with page numbers — all inside a modern Next.js frontend with JWT-secured per-user data isolation. - +The system uses **semantic search + cross-encoder reranking** to find the most relevant document chunks, streams AI-generated answers token-by-token, and highlights exact source citations with page numbers — all inside a clean Flask-served UI with session-based per-user data isolation.
## 🏗️ Architecture -> Contributor note: see [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for a route-by-route system map, request-flow diagrams, and Swagger/OpenAPI documentation guidance. +> Contributor note: see [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for a +> route-by-route system map, request-flow diagrams, ownership boundaries, and +> Swagger/OpenAPI documentation guidance. ```mermaid -flowchart TD - -subgraph group_backend["Backend"] - node_backend_main["App entry
FastAPI app
[main.py]"] - node_backend_routes["Routes
API layer"] - node_backend_auth["Auth
JWT/bcrypt
[auth.py]"] - node_backend_documents["Documents
doc lifecycle
[documents.py]"] - node_backend_chat["Chat
streaming RAG
[chat.py]"] - node_backend_workspaces["Workspaces
tenant scope
[workspaces.py]"] - node_backend_graph_api["Graph API
knowledge graph
[graph.py]"] - node_backend_health["Health
checks
[health.py]"] - node_backend_admin["Admin
ops API
[admin.py]"] - node_backend_ingestion["Ingestion
async pipeline"] - node_backend_rag_retrieval["Retrieval
hybrid RAG
[retriever.py]"] - node_backend_rag_generation["Generation
prompt/LLM
[prompts.py]"] - node_backend_graph_rag["Graph RAG
entity graph
[graph_builder.py]"] - node_backend_storage_db[("Database
SQLAlchemy/PG
[database.py]")] - node_backend_storage_vector[("Vector store
ChromaDB
[vectorstore.py]")] - node_backend_storage_cache[("Cache
Redis/LRU
[cache.py]")] - node_backend_queue["Queue
Celery/Redis
[celery_app.py]"] -end - -subgraph group_frontend["Frontend"] - node_frontend_app["Next app
App Router"] - node_frontend_chat["Chat UI
stream client"] - node_frontend_documents["Docs UI
document workspace"] - node_frontend_graph["Graph view
knowledge graph
[KnowledgeGraph.tsx]"] - node_frontend_state["State
Zustand stores"] - node_frontend_api["API client
fetch layer
[api.ts]"] -end - -subgraph group_ops["Ops & Integrations"] - node_integrations_bot["Discord bot
integration
[bot.py]"] - node_ops_obs["Observability
metrics/tracing
[observability.py]"] - node_ops_deploy["Deploy stack
containers
[docker-compose.yml]"] -end - -node_frontend_app -->|"uses"| node_frontend_api -node_frontend_api -->|"requests"| node_backend_main -node_frontend_chat -->|"reads/writes"| node_frontend_state -node_frontend_documents -->|"reads/writes"| node_frontend_state -node_frontend_graph -->|"loads"| node_frontend_api -node_backend_main -->|"mounts"| node_backend_routes -node_backend_routes -->|"exposes"| node_backend_auth -node_backend_routes -->|"exposes"| node_backend_documents -node_backend_routes -->|"exposes"| node_backend_chat -node_backend_routes -->|"exposes"| node_backend_workspaces -node_backend_routes -->|"exposes"| node_backend_graph_api -node_backend_routes -->|"exposes"| node_backend_health -node_backend_routes -->|"exposes"| node_backend_admin -node_backend_documents -->|"enqueues"| node_backend_queue -node_backend_queue -->|"runs"| node_backend_ingestion -node_backend_ingestion -->|"persists"| node_backend_storage_db -node_backend_ingestion -->|"indexes"| node_backend_storage_vector -node_backend_ingestion -->|"caches"| node_backend_storage_cache -node_backend_chat -->|"retrieves"| node_backend_rag_retrieval -node_backend_rag_retrieval -->|"searches"| node_backend_storage_vector -node_backend_rag_retrieval -->|"filters"| node_backend_storage_db -node_backend_rag_retrieval -->|"supplies context"| node_backend_rag_generation -node_backend_rag_generation -->|"streams"| node_backend_chat -node_backend_graph_api -->|"queries"| node_backend_graph_rag -node_backend_graph_rag -->|"persists"| node_backend_storage_db -node_backend_auth -->|"authenticates"| node_backend_storage_db -node_backend_workspaces -->|"scopes"| node_backend_storage_db -node_backend_health -->|"checks"| node_backend_storage_db -node_backend_health -->|"checks"| node_backend_storage_cache -node_backend_health -->|"checks"| node_backend_queue -node_backend_health -->|"checks"| node_backend_storage_vector -node_ops_obs -->|"observes"| node_backend_main -node_ops_deploy -->|"runs"| node_backend_main -node_ops_deploy -->|"runs"| node_frontend_app -node_integrations_bot -.->|"integrates"| node_backend_routes - -click node_backend_main "https://github.com/param20h/pdf-assistant-rag/blob/dev/backend/app/main.py" -click node_backend_routes "https://github.com/param20h/pdf-assistant-rag/tree/dev/backend/app/routes" -click node_backend_auth "https://github.com/param20h/pdf-assistant-rag/blob/dev/backend/app/routes/auth.py" -click node_backend_documents "https://github.com/param20h/pdf-assistant-rag/blob/dev/backend/app/routes/documents.py" -click node_backend_chat "https://github.com/param20h/pdf-assistant-rag/blob/dev/backend/app/routes/chat.py" -click node_backend_workspaces "https://github.com/param20h/pdf-assistant-rag/blob/dev/backend/app/routes/workspaces.py" -click node_backend_graph_api "https://github.com/param20h/pdf-assistant-rag/blob/dev/backend/app/routes/graph.py" -click node_backend_health "https://github.com/param20h/pdf-assistant-rag/blob/dev/backend/app/routes/health.py" -click node_backend_admin "https://github.com/param20h/pdf-assistant-rag/blob/dev/backend/app/routes/admin.py" -click node_backend_ingestion "https://github.com/param20h/pdf-assistant-rag/blob/dev/backend/app/services/document_ingestion.py" -click node_backend_rag_retrieval "https://github.com/param20h/pdf-assistant-rag/blob/dev/backend/app/rag/retriever.py" -click node_backend_rag_generation "https://github.com/param20h/pdf-assistant-rag/blob/dev/backend/app/rag/prompts.py" -click node_backend_graph_rag "https://github.com/param20h/pdf-assistant-rag/blob/dev/backend/app/rag/graph_builder.py" -click node_backend_storage_db "https://github.com/param20h/pdf-assistant-rag/blob/dev/backend/app/database.py" -click node_backend_storage_vector "https://github.com/param20h/pdf-assistant-rag/blob/dev/backend/app/rag/vectorstore.py" -click node_backend_storage_cache "https://github.com/param20h/pdf-assistant-rag/blob/dev/backend/app/cache.py" -click node_backend_queue "https://github.com/param20h/pdf-assistant-rag/blob/dev/backend/app/celery_app.py" -click node_frontend_app "https://github.com/param20h/pdf-assistant-rag/tree/dev/frontend/src/app" -click node_frontend_chat "https://github.com/param20h/pdf-assistant-rag/tree/dev/frontend/src/components/chat" -click node_frontend_documents "https://github.com/param20h/pdf-assistant-rag/tree/dev/frontend/src/components/document" -click node_frontend_graph "https://github.com/param20h/pdf-assistant-rag/blob/dev/frontend/src/components/graph/KnowledgeGraph.tsx" -click node_frontend_state "https://github.com/param20h/pdf-assistant-rag/tree/dev/frontend/src/store" -click node_frontend_api "https://github.com/param20h/pdf-assistant-rag/blob/dev/frontend/src/lib/api.ts" -click node_integrations_bot "https://github.com/param20h/pdf-assistant-rag/blob/dev/bots/discord/bot.py" -click node_ops_obs "https://github.com/param20h/pdf-assistant-rag/blob/dev/backend/app/observability.py" -click node_ops_deploy "https://github.com/param20h/pdf-assistant-rag/blob/dev/docker-compose.yml" - -classDef toneNeutral fill:#f8fafc,stroke:#334155,stroke-width:1.5px,color:#0f172a -classDef toneBlue fill:#dbeafe,stroke:#2563eb,stroke-width:1.5px,color:#172554 -classDef toneAmber fill:#fef3c7,stroke:#d97706,stroke-width:1.5px,color:#78350f -classDef toneMint fill:#dcfce7,stroke:#16a34a,stroke-width:1.5px,color:#14532d -classDef toneRose fill:#ffe4e6,stroke:#e11d48,stroke-width:1.5px,color:#881337 -classDef toneIndigo fill:#e0e7ff,stroke:#4f46e5,stroke-width:1.5px,color:#312e81 -classDef toneTeal fill:#ccfbf1,stroke:#0f766e,stroke-width:1.5px,color:#134e4a -class node_backend_main,node_backend_routes,node_backend_auth,node_backend_documents,node_backend_chat,node_backend_workspaces,node_backend_graph_api,node_backend_health,node_backend_admin,node_backend_ingestion,node_backend_rag_retrieval,node_backend_rag_generation,node_backend_graph_rag,node_backend_storage_db,node_backend_storage_vector,node_backend_storage_cache,node_backend_queue toneBlue -class node_frontend_app,node_frontend_chat,node_frontend_documents,node_frontend_graph,node_frontend_state,node_frontend_api toneAmber -class node_integrations_bot,node_ops_obs,node_ops_deploy toneMint +graph TD + subgraph Frontend["Frontend (Next.js 16)"] + UI["Dashboard UI (React)"] + Chat["Chat Panel (SSE)"] + Viewer["PDF Viewer (iframe)"] + end + + subgraph Backend["Backend (FastAPI 0.115+)"] + API["API Router (/api/v1)"] + Auth["Auth (JWT/bcrypt)"] + DB[(SQLite Metadata)] + + subgraph RAG["RAG Pipeline"] + Upload["Ingestion Task (Chunking)"] + Embed["Local Embeddings (all-MiniLM-L6-v2)"] + Retriever["Two-Stage Retriever"] + Rerank["Cross-Encoder Reranker"] + Agent["Agent/Generator"] + end + end + + subgraph Storage["Vector Storage"] + Chroma[(ChromaDB)] + end + + subgraph External["External Services"] + HF["HuggingFace Inference API (Qwen 72B)"] + end + + %% Frontend to Backend Connections + UI <-->|REST / Auth| API + Chat <-->|SSE Streaming| API + Viewer -->|Fetch PDF| API + + %% Backend Internals + API <--> Auth + API <--> DB + API --> Upload + API <--> Retriever + API <--> Agent + + %% RAG Ingestion Flow + Upload --> Embed + Embed -->|Store Vectors| Chroma + + %% RAG Query Flow + Retriever -->|1. Semantic Search| Chroma + Retriever -->|2. Score & Sort| Rerank + Retriever -->|Context| Agent + + %% External LLM Flow + Agent <-->|LLM Generation| HF ```
### 🔄 System Flow Overview -1. User uploads a document via the Next.js frontend. -2. FastAPI queues a Celery ingestion task backed by Redis. -3. The worker chunks the document, generates local embeddings (cached via Redis/LRU), builds a BM25 index, and stores vectors in ChromaDB. -4. At query time, hybrid search merges vector and BM25 results via Reciprocal Rank Fusion. -5. A cross-encoder reranker refines the top candidates. -6. The agent assembles a prompt and calls the HuggingFace Inference API. -7. Streamed SSE tokens are returned to the frontend chat panel. +1. The user interacts with the Next.js frontend to upload documents and ask questions. +2. FastAPI handles authentication, document ingestion, and chat APIs. +3. Uploaded documents are parsed, chunked, and converted into vector embeddings. +4. Embeddings are stored in ChromaDB for semantic retrieval. +5. During querying, the retriever fetches relevant chunks from ChromaDB. +6. A reranker improves retrieval quality before sending context to the LLM. +7. Hugging Face Inference API generates the final response. +8. Responses are streamed back to the frontend using SSE.
@@ -225,41 +180,37 @@ class node_integrations_bot,node_ops_obs,node_ops_deploy toneMint | | Technology | Purpose | |---|---|---| -| | **FastAPI** | Async web framework + routing | +| | **Flask 2.x** | Web framework + routing | | | **Python 3.11** | Runtime environment | -| | **PostgreSQL / SQLite** | Relational database (SQLAlchemy ORM) | -| | **JWT + bcrypt** | Authentication & password hashing | -| | **ChromaDB** | Local vector store (embeddings) | -| | **HuggingFace Inference API** | LLM answer generation | -| | **sentence-transformers** | Local embedding model (all-MiniLM-L6-v2) | +| | **MongoDB + PyMongo** | User accounts & metadata storage | +| | **Flask-Login + Flask-Dance** | Session auth + Google OAuth | +| | **Pinecone** | Vector store (per-user namespace) | +| | **Google Gemini API** | Document chunk embeddings | +| | **Groq (Llama 3)** | LLM answer generation | ### Frontend | | Technology | Purpose | |---|---|---| -| | **Next.js 14** | React framework (App Router) | -| | **TypeScript** | Frontend language | -| | **Tailwind CSS** | Utility-first styling | +| | **Jinja2 (via Flask)** | Server-side HTML templating | +| | **HTML + CSS + JavaScript** | Frontend UI (served from `/static` and `/templates`) | ### AI / ML Pipeline | | Technology | Purpose | |---|---|---| -| | **sentence-transformers (all-MiniLM-L6-v2)** | Generates vector embeddings for document chunks | -| | **ChromaDB** | Stores + retrieves embeddings locally | -| | **HuggingFace Inference API** | Generates answers from retrieved context | -| | **BAAI/bge-reranker-v2-m3** | Cross-encoder reranking for retrieval quality | -| | **Knowledge Graph (GraphRAG)** | Entity extraction + relationship graphs | +| | **Google Gemini API** | Generates vector embeddings for document chunks | +| | **Pinecone** | Stores + retrieves embeddings per user namespace | +| | **Groq API (Llama 3)** | Generates answers from retrieved context | | | **PyMuPDF + pdfplumber + python-docx** | Document text extraction | ### DevOps & Tooling | | Technology | Purpose | |---|---|---| -| | **Docker Multi-Stage** | Containerised deployment | -| | **GitHub Actions** | CI/CD (E2E, security, deploy) | -| | **Playwright** | E2E + visual regression tests | -| | **Prometheus + Grafana** | Metrics & observability | +| | **Docker Multi-Stage** | Containerized deployment | +| | **GitHub Actions** | CI pipeline (dev branch) | +| | **Git LFS** | Binary asset management | | | **HuggingFace Spaces** | Production deployment | @@ -268,27 +219,16 @@ class node_integrations_bot,node_ops_obs,node_ops_deploy toneMint ## ✨ Key Features -### 🆕 Recent Updates - -- 🤖 Discord Bot Integration -- ⚡ Celery + Redis Background PDF Processing -- 📧 Email Verification Workflow -- 🧠 RAGAS Evaluation Pipeline -- 🚀 Response Caching with Redis -- 🐳 Optimized Docker Deployment - @@ -297,25 +237,21 @@ class node_integrations_bot,node_ops_obs,node_ops_deploy toneMint ### 🤖 RAG Pipeline - 🔪 Smart **recursive text chunking** (configurable size & overlap) - 🧠 **Local embeddings** — no data leaves your machine -- ⚡ **Embedding cache** (Redis + LRU) — skip redundant computation -- 🔍 **Hybrid search** — vector + BM25 merged via RRF -- 🏆 **Cross-encoder reranking** for precision answers -- 🖼️ **Image caption extraction** from PDF figures -- 🔗 **URL extraction** from PDF link annotations -- 🗺️ **Knowledge graph** (GraphRAG) per document +- 🔍 **Two-stage retrieval** — semantic search → cross-encoder rerank +- ✂️ Top-K filtering for precision answers +- 📝 Custom **system prompts** with citation instructions +- 🧾 Source scoring with confidence levels @@ -328,69 +264,36 @@ class node_integrations_bot,node_ops_obs,node_ops_deploy toneMint ``` PDF-Assistant-RAG/ │ -├── backend/ -│ ├── app/ -│ │ ├── main.py # FastAPI app — lifespan, middleware, routers -│ │ ├── config.py # Pydantic settings (env vars) -│ │ ├── models.py # SQLAlchemy ORM models -│ │ ├── schemas.py # Pydantic request/response schemas -│ │ ├── database.py # Engine, session, migrations -│ │ ├── auth.py # JWT helpers -│ │ ├── tasks.py # Celery task definitions -│ │ │ -│ │ ├── routes/ -│ │ │ ├── auth.py # Register, login, OAuth -│ │ │ ├── documents.py # Upload, list, delete, status -│ │ │ ├── chat.py # Ask, stream, history, export -│ │ │ ├── health.py # Deep health check endpoint -│ │ │ ├── admin.py # Admin stats -│ │ │ └── workspaces.py # Workspace management -│ │ │ -│ │ ├── rag/ -│ │ │ ├── chunker.py # PDF/DOCX/TXT extraction + chunking -│ │ │ ├── embeddings.py # Local embeddings + Redis/LRU cache -│ │ │ ├── vectorstore.py # ChromaDB operations -│ │ │ ├── bm25.py # BM25 index per document -│ │ │ ├── retriever.py # Hybrid search + RRF + reranking -│ │ │ ├── reranker.py # Cross-encoder reranker -│ │ │ ├── vision.py # Image caption extraction -│ │ │ ├── url_extractor.py # PDF URL/link extraction -│ │ │ ├── graph_builder.py # Knowledge graph (GraphRAG) -│ │ │ ├── agent.py # LLM answer generation -│ │ │ └── summarizer.py # Document summarisation -│ │ │ -│ │ └── services/ -│ │ └── document_ingestion.py # End-to-end ingestion pipeline -│ │ -│ ├── tests/ # pytest test suite -│ ├── requirements.txt -│ └── migrate_add_extracted_urls.py +├── app.py # Flask app — all routes (upload, ask, download, auth) +├── config.py # Loads SECRET_KEY, MONGO_URI, Google OAuth credentials +├── models.py # User model (MongoDB via PyMongo) +├── make_admin.py # CLI script to promote a user to admin │ -├── frontend/ -│ ├── src/ -│ │ ├── app/ # Next.js App Router pages -│ │ ├── components/ # React components -│ │ ├── store/ # Zustand state stores -│ │ ├── lib/ # API client, auth, utilities -│ │ └── services/ # API service layer -│ ├── e2e/ # Playwright E2E + snapshot tests -│ ├── next.config.ts -│ └── playwright.config.ts +├── rag/ +│ ├── chunker.py # Splits PDF/DOCX/TXT into text chunks +│ ├── embeddings.py # Gemini embeddings → Pinecone store/delete +│ ├── retriever.py # Pinecone similarity search → top-K chunks +│ └── generator.py # Groq LLM → answer from retrieved context │ -├── docs/ -│ └── ARCHITECTURE.md +├── static/ # CSS, JS, images +├── templates/ # Jinja2 HTML templates (login, register, chat, admin) +├── uploads/ # Per-user uploaded files (gitignored) +├── instance/ # Flask instance folder │ ├── .github/ -│ └── workflows/ -│ ├── ci.yml # Backend CI -│ ├── e2e.yml # Playwright E2E + visual regression -│ ├── deploy.yml # Docker build (main branch) -│ └── devsecops.yml # Security scans +│ ├── workflows/ +│ │ ├── ci.yml # CI — runs on dev branch only +│ │ ├── deploy.yml # Docker build — main branch only +│ │ └── devsecops.yml # Security scans — main branch only +│ ├── ISSUE_TEMPLATE/ # Bug report & feature request forms +│ └── pull_request_template.md │ -├── docker-compose.yml # CPU + GPU + debug profiles + log rotation -├── Dockerfile # Multi-stage backend build -├── frontend/Dockerfile # Multi-stage frontend build (nginx) -└── .env.example +├── .env.example # Template for required environment variables +├── requirements.txt # Python dependencies +├── Dockerfile # Docker build +├── docker-compose.yml # Local Docker stack +├── start.sh # Gunicorn startup script +└── render.yaml # Render.com deployment config ```
@@ -400,9 +303,10 @@ PDF-Assistant-RAG/ ### Prerequisites - ![Python](https://img.shields.io/badge/Python-3.11+-3776AB?style=flat&logo=python&logoColor=white) **Python 3.11+** -- ![Node](https://img.shields.io/badge/Node.js-20+-339933?style=flat&logo=nodedotjs&logoColor=white) **Node.js 20+** -- ![Docker](https://img.shields.io/badge/Docker-2496ED?style=flat&logo=docker&logoColor=white) **Docker + Docker Compose** (recommended) -- ![HuggingFace](https://img.shields.io/badge/HuggingFace_Token-FFD21E?style=flat&logo=huggingface&logoColor=black) **HuggingFace API token** — [huggingface.co/settings/tokens](https://huggingface.co/settings/tokens) (free) +- ![MongoDB](https://img.shields.io/badge/MongoDB-Atlas-47A248?style=flat&logo=mongodb&logoColor=white) **MongoDB** (Atlas free tier or local) +- ![Pinecone](https://img.shields.io/badge/Pinecone-000000?style=flat) **Pinecone** account — [pinecone.io](https://pinecone.io) +- ![Gemini](https://img.shields.io/badge/Google_Gemini-4285F4?style=flat&logo=google&logoColor=white) **Google Gemini API key** — [aistudio.google.com](https://aistudio.google.com) +- ![Groq](https://img.shields.io/badge/Groq-F55036?style=flat) **Groq API key** — [console.groq.com](https://console.groq.com) --- @@ -417,13 +321,11 @@ cd PDF-Assistant-RAG ```bash cp .env.example .env -``` Edit `.env`: -```env SECRET_KEY=your-strong-random-secret -DATABASE_URL=postgresql://pdf_rag_user:pdf_rag_pass@localhost:5432/pdf_rag +DATABASE_URL=sqlite:///./data/app.db HF_TOKEN=hf_your_huggingface_token_here UPLOAD_DIR=./data/uploads CHROMA_PERSIST_DIR=./data/chroma_db @@ -433,64 +335,63 @@ CELERY_RESULT_BACKEND=redis://localhost:6379/1 > Get your free HuggingFace token at [huggingface.co/settings/tokens](https://huggingface.co/settings/tokens) -#### Email Verification Setup (optional) +#### Email Verification Setup + +Password registration requires email verification before users can log in. To send real verification emails, add SMTP settings to `backend/.env`: ```env FRONTEND_URL=http://localhost:3000 +EMAIL_VERIFICATION_TOKEN_EXPIRE_HOURS=24 MAIL_USERNAME=your_smtp_username MAIL_PASSWORD=your_smtp_or_gmail_app_password MAIL_FROM=your_sender_email@example.com -MAIL_SERVER=smtp.gmail.com +MAIL_SERVER=smtp.example.com MAIL_PORT=587 MAIL_STARTTLS=True MAIL_SSL_TLS=False ``` -Without SMTP settings, registration returns a local verification link so contributors can test without email credentials. +For Gmail, enable 2-Step Verification on the sender Google account, create a 16-character App Password from Google Account > Security > App passwords, then use: + +```env +MAIL_USERNAME=yourgmail@gmail.com +MAIL_PASSWORD=your_16_character_app_password +MAIL_FROM=yourgmail@gmail.com +MAIL_SERVER=smtp.gmail.com +MAIL_PORT=587 +MAIL_STARTTLS=True +MAIL_SSL_TLS=False +``` -### 3. Run with Docker (recommended) +Without SMTP settings in a non-production environment, registration returns a local verification link so contributors can test the flow without private email credentials. With SMTP configured, the same link is sent by email. -```bash -# CPU-only (no GPU needed) -docker compose --profile cpu up --build +### 3. Set up crawl4ai (URL Upload Feature) -# GPU-accelerated (requires NVIDIA Container Toolkit) -docker compose --profile gpu up --build +The URL upload feature (`POST /api/v1/documents/urlupload`) uses **crawl4ai** with a Playwright browser to crawl web pages. `crawl4ai-setup` handles the Playwright browser installation automatically — run it once after `pip install`: -# Also start pgAdmin at http://localhost:5050 -docker compose --profile cpu --profile debug up --build +```bash +crawl4ai-setup ``` -| Service | URL | -|---------|-----| -| Frontend | http://localhost:3000 | -| Backend API | http://localhost:7860 | -| API Docs | http://localhost:7860/docs | -| pgAdmin | http://localhost:5050 (debug profile) | -### 4. Run Locally (without Docker) +--- + +### 3. Run Locally ```bash -# Backend -cd backend +# Single terminal — Flask app python -m venv .venv source .venv/bin/activate # Windows: .venv\Scripts\activate pip install -r requirements.txt -uvicorn app.main:app --reload --port 7860 - -# Celery worker (separate terminal) -celery -A app.celery_app.celery_app worker --loglevel=info - -# Frontend (separate terminal) -cd frontend -npm install -npm run dev +python app.py +# → App running at http://localhost:5000 ``` -### 5. Set up crawl4ai (URL Upload Feature — optional) +### 4. Run with Docker ```bash -crawl4ai-setup +docker compose up --build +# → App running at http://localhost:7860 ```
@@ -499,55 +400,48 @@ crawl4ai-setup ``` ┌─────────────────────────────────────────────┐ - │ PDF / DOCX / TXT / MD Upload │ + │ PDF / DOCX Upload │ + └───────────────────┬─────────────────────────┘ + │ + ▼ + ┌─────────────────────────────────────────────┐ + │ PyMuPDF / python-docx Parser │ + │ (text extraction per page) │ └───────────────────┬─────────────────────────┘ │ ▼ ┌─────────────────────────────────────────────┐ - │ PyMuPDF / pdfplumber / python-docx Parser │ - │ + Image caption extraction │ - │ + PDF URL/link annotation extraction │ + │ Recursive Character Text Splitter │ + │ chunk_size=1000 | overlap=200 │ └───────────────────┬─────────────────────────┘ │ ▼ ┌─────────────────────────────────────────────┐ - │ Recursive Character Text Splitter │ - │ chunk_size=1000 | overlap=200 │ + │ all-MiniLM-L6-v2 (local embeddings) │ + │ 384-dim dense vectors │ └───────────────────┬─────────────────────────┘ │ ▼ ┌─────────────────────────────────────────────┐ - │ all-MiniLM-L6-v2 (local embeddings) │ - │ 384-dim · Redis + LRU cache (24h TTL) │ - └──────────────┬──────────────────────────────┘ - │ - ┌──────────────┴──────────────┐ - ▼ ▼ - ┌──────────────────┐ ┌─────────────────────┐ - │ ChromaDB vectors │ │ BM25 keyword index │ - │ (per-user coll.) │ │ (per-document .pkl)│ - └──────────────────┘ └─────────────────────┘ - - ── At Query Time ── - - User Question ──▶ Embed (cached) ──▶ Vector Search (Top-K=20) - │ - ├──▶ BM25 Search (Top-K=20) - │ - ▼ - Reciprocal Rank Fusion (RRF, k=60) - │ - ▼ - BGE-Reranker-v2-m3 Cross-Encoder (Top-K=8) - │ - ▼ - Prompt Assembly (system + context + question) - │ - ▼ - Qwen2.5-72B-Instruct (HF Inference API) - │ - ▼ - Streamed SSE tokens ──▶ Frontend ChatPanel + │ ChromaDB — per-user persistent collection │ + └─────────────────────────────────────────────┘ + + ── At Query Time ── + + User Question ──▶ Embed ──▶ Semantic Search (Top-K=10) + │ + ▼ + Cross-Encoder Reranker (Top-K=5) + ms-marco-MiniLM-L-6-v2 + │ + ▼ + Prompt Assembly (system + context + question) + │ + ▼ + Qwen2.5-72B-Instruct (HF Inference API) + │ + ▼ + Streamed SSE tokens ──▶ Frontend ChatPanel ```
@@ -557,55 +451,63 @@ crawl4ai-setup | Method | Endpoint | Auth | Description | |--------|----------|------|-------------| | `POST` | `/api/v1/auth/register` | ❌ | Create a new user account | -| `POST` | `/api/v1/auth/login` | ❌ | Login and receive JWT tokens | +| `POST` | `/api/v1/auth/login` | ❌ | Login and receive JWT token | | `GET` | `/api/v1/auth/me` | ✅ | Get current user profile | -| `POST` | `/api/v1/documents/upload` | ✅ | Upload PDF/DOCX/TXT and enqueue ingestion (`202`) | -| `POST` | `/api/v1/documents/urlupload` | ✅ | Crawl a URL and ingest as document | -| `GET` | `/api/v1/documents/` | ✅ | List documents (pagination + `?q=` name filter) | -| `GET` | `/api/v1/documents/{id}` | ✅ | Get document metadata (incl. extracted URLs) | -| `GET` | `/api/v1/documents/{id}/status` | ✅ | Poll ingestion progress | -| `DELETE` | `/api/v1/documents/{id}` | ✅ | Soft-delete document | -| `POST` | `/api/v1/chat/ask/stream` | ✅ | Ask a question (SSE streaming) | +| `POST` | `/api/v1/documents/upload` | ✅ | Upload PDF/DOCX and enqueue background indexing (`202 Accepted`) | +| `GET` | `/api/v1/documents` | ✅ | List all documents for current user | +| `GET` | `/api/v1/documents/{id}/status` | ✅ | Poll background document processing status | +| `DELETE` | `/api/v1/documents/{id}` | ✅ | Delete a document and its vector data | +| `POST` | `/api/v1/chat/ask/stream` | ✅ | Ask a question (SSE streaming response) | | `GET` | `/api/v1/chat/history/{doc_id}` | ✅ | Get chat history for a document | -| `DELETE` | `/api/v1/chat/history/{doc_id}` | ✅ | Clear chat history | -| `GET` | `/api/v1/chat/export/{doc_id}` | ✅ | Export transcript as MD / TXT / PDF | -| `GET` | `/api/v1/chat/sessions` | ✅ | List chat sessions | -| `POST` | `/api/v1/chat/sessions` | ✅ | Create chat session | -| `GET` | `/api/v1/health/status` | ❌ | Deep health check (DB, Redis, Celery, ChromaDB) | -| `GET` | `/api/health` | ❌ | Basic liveness check | +| `DELETE` | `/api/v1/chat/history/{doc_id}` | ✅ | Clear chat history for a document | +| `GET` | `/health` | ❌ | Health check (db + chroma status) | -> Full interactive docs at `/docs` (Swagger UI) when running locally. +> Full interactive docs available at `/docs` (Swagger UI) when running locally.
## 📦 Environment Variables -| Variable | Required | Default | Description | -|---|---|---|---| -| `SECRET_KEY` | ✅ | — | JWT signing secret. Generate: `python -c "import secrets; print(secrets.token_urlsafe(32))"` | -| `HF_TOKEN` | ✅ | — | HuggingFace API token for LLM inference | -| `DATABASE_URL` | ❌ | `sqlite:///./data/app.db` | SQLAlchemy connection string (SQLite or PostgreSQL) | -| `CELERY_BROKER_URL` | ❌ | `redis://localhost:6379/0` | Redis broker for Celery | -| `CELERY_RESULT_BACKEND` | ❌ | `redis://localhost:6379/1` | Redis backend for Celery results | -| `REDIS_URL` | ❌ | — | Redis URL for response + embedding cache | -| `UPLOAD_DIR` | ❌ | `./data/uploads` | File storage directory | -| `CHROMA_PERSIST_DIR` | ❌ | `./data/chroma_db` | ChromaDB persistence directory | -| `EMBEDDING_MODEL` | ❌ | `sentence-transformers/all-MiniLM-L6-v2` | Local embedding model | -| `EMBEDDING_CACHE_TTL` | ❌ | `86400` | Embedding cache TTL in seconds (24h) | -| `LLM_MODEL` | ❌ | `Qwen/Qwen2.5-72B-Instruct` | HuggingFace model for answer generation | -| `LLM_TEMPERATURE` | ❌ | `0.3` | LLM sampling temperature | -| `RERANKER_MODEL` | ❌ | `BAAI/bge-reranker-v2-m3` | Cross-encoder reranker model | -| `USE_HYBRID_SEARCH` | ❌ | `True` | Enable BM25 + vector hybrid search | -| `RRF_K` | ❌ | `60` | RRF smoothing constant | -| `CHUNK_SIZE` | ❌ | `1000` | Characters per document chunk | -| `CHUNK_OVERLAP` | ❌ | `200` | Overlap between consecutive chunks | -| `TOP_K_RETRIEVAL` | ❌ | `20` | Candidates retrieved from vector store | -| `TOP_K_RERANK` | ❌ | `8` | Final chunks after reranking | -| `VISION_PROVIDER` | ❌ | — | Set to `openai` to use GPT-4o-mini for image captions | -| `OPENAI_API_KEY` | ❌ | — | Required when `VISION_PROVIDER=openai` | -| `ENVIRONMENT` | ❌ | `development` | Set to `production` to lock CORS | -| `FRONTEND_URL` | ❌ | `http://localhost:3000` | Public frontend URL for OAuth + email links | -| `NEXT_PUBLIC_API_URL` | ❌ | `http://localhost:7860` | Backend URL injected at frontend build time | +| Variable | Required | Default | Description | Where to Get It | +|---|---|---|---|---| +| `SECRET_KEY` | ✅ | — | JWT signing & session secret. Use a strong random string. | Generate: `python -c "import secrets; print(secrets.token_urlsafe(32))"` | +| `HF_TOKEN` | ✅ | — | HuggingFace API token for LLM inference via Inference API. | [huggingface.co/settings/tokens](https://huggingface.co/settings/tokens) (free) | +| `HF_CLIENT_ID` | ❌ | — | HuggingFace OAuth client ID. Required only for Hugging Face sign-in. | [HuggingFace Developer Settings](https://huggingface.co/settings/connected-applications) | +| `HF_CLIENT_SECRET` | ❌ | — | HuggingFace OAuth client secret. Required only for Hugging Face sign-in. | [HuggingFace Developer Settings](https://huggingface.co/settings/connected-applications) | +| `HF_REDIRECT_URI` | ❌ | `http://localhost:8000/api/v1/auth/callback/huggingface` | HuggingFace OAuth callback redirect URI. | — | +| `FRONTEND_URL` | ❌ | `http://localhost:3000` | Public frontend URL used for OAuth redirects and email verification links. | Your deployed frontend URL | +| `ENVIRONMENT` | ❌ | `development` | Runtime mode. Set to `production` for deployment to lock CORS. | — | +| `DEBUG` | ❌ | `False` | Enable debug mode with detailed error pages. Never enable in production. | — | +| `ALLOWED_ORIGINS` | ❌ | `http://localhost:3000,http://localhost:7860` | Comma-separated CORS origins (only enforced in production). | Your deployed domain(s) | +| `DATABASE_URL` | ❌ | `sqlite:///./data/app.db` | SQLAlchemy database connection string. | SQLite (default), or your Postgres/MySQL connection string | +| `JWT_ALGORITHM` | ❌ | `HS256` | JWT signing algorithm. | — | +| `JWT_EXPIRY_HOURS` | ❌ | `72` | JWT token lifetime in hours before re-login is required. | — | +| `GOOGLE_CLIENT_ID` | ❌ | — | Google OAuth web client ID used by FastAPI to verify ID tokens. | [Google Cloud Console](https://console.cloud.google.com/apis/credentials) | +| `NEXT_PUBLIC_GOOGLE_CLIENT_ID` | ❌ | — | Google OAuth web client ID exposed to the Next.js Google sign-in button. | [Google Cloud Console](https://console.cloud.google.com/apis/credentials) | +| `EMAIL_VERIFICATION_TOKEN_EXPIRE_HOURS` | ❌ | `24` | Email verification token lifetime in hours. | — | +| `MAIL_USERNAME` | ❌ | — | SMTP username for account verification emails. | SMTP provider or Gmail App Password setup | +| `MAIL_PASSWORD` | ❌ | — | SMTP password or Gmail 16-character App Password. | SMTP provider or Gmail App Password setup | +| `MAIL_FROM` | ❌ | — | Sender email address for verification emails. | Verified sender address | +| `MAIL_SERVER` | ❌ | — | SMTP server hostname, for example `smtp.gmail.com`. | SMTP provider | +| `MAIL_PORT` | ❌ | `587` | SMTP server port. | SMTP provider | +| `MAIL_STARTTLS` | ❌ | `True` | Enable STARTTLS for SMTP. | SMTP provider | +| `MAIL_SSL_TLS` | ❌ | `False` | Enable SSL/TLS for SMTP. | SMTP provider | +| `CELERY_BROKER_URL` | ❌ | `redis://localhost:6379/0` | Redis broker URL used by FastAPI to queue document ingestion jobs. | Redis | +| `CELERY_RESULT_BACKEND` | ❌ | `redis://localhost:6379/1` | Redis backend URL used by Celery to store task state/results. | Redis | +| `UPLOAD_DIR` | ❌ | `./data/uploads` | Local directory for storing uploaded documents. | — | +| `MAX_FILE_SIZE_MB` | ❌ | `50` | Maximum allowed upload file size in MB. | — | +| `ALLOWED_EXTENSIONS` | ❌ | `pdf,docx,txt,md` | Comma-separated list of permitted file extensions. | — | +| `CHROMA_PERSIST_DIR` | ❌ | `./data/chroma_db` | Directory where ChromaDB persists its vector index. | — | +| `LLM_MODEL` | ❌ | `Qwen/Qwen2.5-72B-Instruct` | HuggingFace model ID for answer generation. | [huggingface.co/models](https://huggingface.co/models?inference=warm&sort=trending) | +| `LLM_TEMPERATURE` | ❌ | `0.3` | LLM sampling temperature (0 = deterministic, 1 = creative). | — | +| `LLM_MAX_NEW_TOKENS` | ❌ | `1024` | Maximum tokens per LLM response. | — | +| `EMBEDDING_MODEL` | ❌ | `sentence-transformers/all-MiniLM-L6-v2` | SentenceTransformer model for local embeddings (no external API). | [huggingface.co/sentence-transformers](https://huggingface.co/sentence-transformers) | +| `EMBEDDING_DIMENSION` | ❌ | `384` | Embedding vector dimension (must match the model). | — | +| `RERANKER_MODEL` | ❌ | `cross-encoder/ms-marco-MiniLM-L-6-v2` | Cross-encoder model for reranking retrieved chunks by relevance. | [huggingface.co/cross-encoder](https://huggingface.co/cross-encoder) | +| `CHUNK_SIZE` | ❌ | `1000` | Characters per document chunk. Larger = more context, smaller = better precision. | — | +| `CHUNK_OVERLAP` | ❌ | `200` | Overlap between consecutive chunks to maintain boundary context. | — | +| `TOP_K_RETRIEVAL` | ❌ | `10` | Candidate chunks retrieved from vector store during semantic search. | — | +| `TOP_K_RERANK` | ❌ | `5` | Final chunks passed to the LLM after reranking (must be ≤ `TOP_K_RETRIEVAL`). | — |
@@ -616,40 +518,43 @@ crawl4ai-setup | Command | Description | |---------|-------------| | `uvicorn app.main:app --reload` | Start FastAPI with hot reload | -| `celery -A app.celery_app.celery_app worker --loglevel=info` | Start Celery worker | -| `python migrate_add_extracted_urls.py` | Run URL extraction column migration | -| `python scripts/run_ragas_eval.py --user-id ` | Run RAGAS evaluation (vector vs GraphRAG) | +| `uvicorn app.main:app --port 8000` | Start FastAPI on port 8000 | +| `python scripts/run_ragas_eval.py --user-id ` | Run the 50-question RAGAS comparison for vector search vs GraphRAG | + +The RAGAS script reads `backend/evaluation/ragas_sample_questions.jsonl`, +generates answers from standard vector contexts and vector-plus-GraphRAG +contexts, then writes aggregate scores to `backend/evaluation/ragas_results.json`. +Pass `--document-id ` to evaluate one indexed document. ### Frontend (`frontend/`) | Command | Description | |---------|-------------| -| `npm run dev` | Start Next.js dev server | -| `npm run build` | Production build | -| `npm run test` | Run Vitest unit tests | -| `npm run test:e2e` | Run Playwright E2E tests | -| `npx playwright test e2e/snapshots.spec.ts --update-snapshots` | Regenerate visual regression baselines | +| `npm run dev` | Start **Next.js** dev server | +| `npm run build` | Production build → `out/` (static export) | +| `npm run lint` | Run ESLint | +| `npm run test:e2e` | Run Playwright end-to-end tests | ### Docker | Command | Description | |---------|-------------| -| `docker compose --profile cpu up --build` | Full stack — CPU only | -| `docker compose --profile gpu up --build` | Full stack — GPU accelerated | -| `docker compose --profile debug up` | Also start pgAdmin at http://localhost:5050 | +| `docker compose up --build` | Build and start the full stack | | `docker compose down` | Stop all containers | -> **GPU profile** requires [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). -
## 🌐 Deployment +This project is deployed on **HuggingFace Spaces** using Docker. + ### HuggingFace Spaces 1. Fork this repo and create a new Space at [huggingface.co/new-space](https://huggingface.co/new-space) (SDK: Docker) -2. Set Space secrets: `HF_TOKEN`, `SECRET_KEY`, `DATABASE_URL` -3. Push to the `hf` remote: +2. Set the following Space secrets: + - `HF_TOKEN` — your HuggingFace API token + - `SECRET_KEY` — a strong random string +3. Push to the `hf` remote — the Space will auto-build ```bash git remote add hf https://:@huggingface.co/spaces// @@ -659,9 +564,8 @@ git push hf main ### Self-Hosted / VPS ```bash -docker compose --profile cpu up -d --build -# App at http://your-server:7860 -# Frontend at http://your-server:3000 +docker compose up -d --build +# App available at http://your-server:7860 ```
@@ -700,13 +604,13 @@ Distributed under the **MIT License**. See [`LICENSE`](license) for more informa
-**Built with 💙 by the open-source community** +**Built with 💙 as a flagship AI engineering project** *If you found this project helpful, please give it a ⭐ — it helps contributors discover it!*
-[![Stack](https://skillicons.dev/icons?i=fastapi,python,react,nextjs,docker)](https://skillicons.dev) +[![Stack](https://skillicons.dev/icons?i=flask,python,mongodb,docker)](https://skillicons.dev)
diff --git a/app.py b/app.py new file mode 100644 index 00000000..92313179 --- /dev/null +++ b/app.py @@ -0,0 +1,450 @@ +from werkzeug.utils import secure_filename +import os +os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1" +os.environ["OAUTHLIB_RELAX_TOKEN_SCOPE"] = "1" +os.environ["REQUESTS_CA_BUNDLE"] = "" +os.environ["CURL_CA_BUNDLE"] = "" + +import ssl +ssl._create_default_https_context = ssl._create_unverified_context + +import urllib3 +urllib3.disable_warnings() + +# ── Patch HTTPAdapter BEFORE any other imports ──────── +import requests +original_send = requests.adapters.HTTPAdapter.send +def patched_send(self, request, **kwargs): + kwargs["verify"] = False + return original_send(self, request, **kwargs) +requests.adapters.HTTPAdapter.send = patched_send + +# ── Patch OAuth2Session fetch_token ─────────────────── +from requests_oauthlib import OAuth2Session +original_fetch = OAuth2Session.fetch_token +def patched_fetch(self, *args, **kwargs): + kwargs["verify"] = False + return original_fetch(self, *args, **kwargs) +OAuth2Session.fetch_token = patched_fetch + +from flask import Flask, request, jsonify, render_template, redirect, url_for, send_file +from flask_login import LoginManager, login_user, logout_user, login_required, current_user +from flask_dance.contrib.google import make_google_blueprint, google +from flask_dance.consumer import oauth_authorized +from dotenv import load_dotenv +from models import User +from rag.chunker import load_and_chunk +from rag.embeddings import store_embeddings, delete_embeddings, clear_all_embeddings +from rag.retriever import retrieve_chunks +from rag.generator import generate_answer +from config import SECRET_KEY, MONGO_URI, GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET + +# ── Init ───────────────────────────────────────────── +load_dotenv() +app = Flask(__name__) + +# Trust reverse proxy headers (Render, Heroku, etc.) +from werkzeug.middleware.proxy_fix import ProxyFix +app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1, x_host=1) + +app.config["SECRET_KEY"] = SECRET_KEY +app.config["UPLOAD_FOLDER"] = "uploads" + +os.makedirs("uploads", exist_ok=True) + +# ── Google Blueprint ────────────────────────────────── +google_bp = make_google_blueprint( + client_id=GOOGLE_CLIENT_ID, + client_secret=GOOGLE_CLIENT_SECRET, + scope=["openid", "profile", "email"], +) +app.register_blueprint(google_bp, url_prefix="/login") + +# ── Database & Login Manager ────────────────────────── +login_manager = LoginManager() +login_manager.init_app(app) +login_manager.login_view = "login" + +chat_history = {} + +@login_manager.user_loader +def load_user(user_id): + return User.get(user_id) + +# ── Google OAuth Signal Handler ─────────────────────── +@oauth_authorized.connect_via(google_bp) +def google_logged_in(blueprint, token): + if not token: + return False + + try: + resp = blueprint.session.get("/oauth2/v2/userinfo") + if not resp.ok: + return False + + google_info = resp.json() + email = google_info.get("email") + name = google_info.get("name", "") + picture = google_info.get("picture","") + username = name.replace(" ", "_").lower() if name else email.split("@")[0] + + if not email: + return False + + with app.app_context(): + user = User.find_by_email(email) + + if not user: + if User.find_by_username(username): + username = username + "_g" + + user = User( + username=username, + email=email, + profile_pic=picture + ) + user.set_password(os.urandom(24).hex()) + user.save() + else: + if picture and user.profile_pic != picture: + user.profile_pic = picture + user.save() + + login_user(user) + + except Exception as e: + print(f"Google login error: {e}") + + return False + +@app.route("/upload_profile_pic", methods=["POST"]) +@login_required +def upload_profile_pic(): + try: + if "profile_pic" not in request.files: + return jsonify({"error": "No file found"}), 400 + + file = request.files["profile_pic"] + + if file.filename == "": + return jsonify({"error": "No file selected"}), 400 + + # ── Check file type ── + allowed = {"png", "jpg", "jpeg", "gif", "webp"} + ext = file.filename.rsplit(".", 1)[1].lower() + if ext not in allowed: + return jsonify({"error": "Only image files allowed"}), 400 + + # ── Save profile pic ── + pic_folder = os.path.join("static", "profile_pics") + os.makedirs(pic_folder, exist_ok=True) + + filename = f"{current_user.username}.{ext}" + filepath = os.path.join(pic_folder, filename) + file.save(filepath) + + current_user.profile_pic = f"/static/profile_pics/{filename}" + current_user.save() + + return jsonify({ + "message": "Profile picture updated!", + "profile_pic": current_user.profile_pic + }), 200 + + except Exception as e: + return jsonify({"error": str(e)}), 500 + + +@app.route("/get_profile", methods=["GET"]) +@login_required +def get_profile(): + return jsonify({ + "username": current_user.username, + "profile_pic": current_user.profile_pic or "" + }), 200 + +# ── Helper Functions ────────────────────────────────── +def allowed_file(filename): + return "." in filename and filename.rsplit(".", 1)[1].lower() in {"pdf", "docx", "txt", "md"} + +def get_user_upload_folder(username): + folder = os.path.join("uploads", username) + os.makedirs(folder, exist_ok=True) + return folder + +def user_has_rag_keys(user): + """Check if user has all required keys for RAG operations.""" + return (user.get_gemini_key() and user.get_pinecone_key() and user.pinecone_index_name) + +# ── Auth Routes ─────────────────────────────────────── + +@app.route("/") +@login_required +def index(): + return render_template("index.html", username=current_user.username) + +@app.route("/register", methods=["GET", "POST"]) +def register(): + if request.method == "POST": + data = request.form + username = data.get("username") + email = data.get("email") + password = data.get("password") + + if User.find_by_username(username): + return render_template("register.html", error="Username already exists!") + + if User.find_by_email(email): + return render_template("register.html", error="Email already exists!") + + user = User(username=username, email=email) + user.set_password(password) + user.save() + return redirect(url_for("login")) + + return render_template("register.html") + +@app.route("/login", methods=["GET", "POST"]) +def login(): + if request.method == "POST": + data = request.form + username = data.get("username") + password = data.get("password") + + user = User.find_by_username(username) + + if not user or not user.check_password(password): + return render_template("login.html", error="Invalid username or password!") + + login_user(user) + return redirect(url_for("index")) + + return render_template("login.html") + +@app.route("/logout") +@login_required +def logout(): + logout_user() + return redirect(url_for("login")) + +# ── App Routes ──────────────────────────────────────── + +@app.route("/chat") +@login_required +def chat(): + return render_template("chat.html", username=current_user.username) + +@app.route("/admin", methods=["GET"]) +@login_required +def admin_dashboard(): + if not current_user.is_admin: + return "Unauthorized", 403 + users = User.get_all() + user_files = {} + for user in users: + folder = get_user_upload_folder(user.username) + if os.path.exists(folder): + user_files[user.username] = [f for f in os.listdir(folder) if f.endswith((".pdf", ".docx", ".txt", ".md"))] + else: + user_files[user.username] = [] + + return render_template("admin.html", users=users, user_files=user_files) + +@app.route("/download//") +@login_required +def download_file(username, filename): + if current_user.username != username and not current_user.is_admin: + return "Unauthorized", 403 + + safe_name = secure_filename(filename) + if not safe_name: + return "Invalid filename", 400 + + folder = get_user_upload_folder(username) + filepath = os.path.join(folder, safe_name) + + # Prevent path traversal even after secure_filename + if not os.path.abspath(filepath).startswith(os.path.abspath(folder)): + return "Forbidden", 403 + + if not os.path.exists(filepath): + return "File not found", 404 + + return send_file(filepath, as_attachment=True) + +@app.route("/profile", methods=["GET"]) +@login_required +def profile(): + return render_template("profile.html", current_user=current_user) + +@app.route("/update_settings", methods=["POST"]) +@login_required +def update_settings(): + try: + data = request.get_json() + current_user.preferred_model = data.get("preferred_model", "groq") + + # ── Groq Key ── + groq_req = data.get("groq_key", "").strip() + if groq_req == "DELETE": + current_user.set_groq_key(None) + elif groq_req: + current_user.set_groq_key(groq_req) + + # ── Gemini Key ── + gemini_req = data.get("gemini_key", "").strip() + if gemini_req == "DELETE": + current_user.set_gemini_key(None) + elif gemini_req: + current_user.set_gemini_key(gemini_req) + + # ── Pinecone Key ── + pinecone_req = data.get("pinecone_key", "").strip() + if pinecone_req == "DELETE": + current_user.set_pinecone_key(None) + elif pinecone_req: + current_user.set_pinecone_key(pinecone_req) + + # ── Pinecone Index Name ── + pinecone_index = data.get("pinecone_index", "").strip() + if pinecone_index: + current_user.pinecone_index_name = pinecone_index + + current_user.save() + return jsonify({"message": "Settings updated successfully!"}), 200 + except Exception as e: + return jsonify({"error": str(e)}), 500 + +@app.route("/files", methods=["GET"]) +@login_required +def get_files(): + try: + folder = get_user_upload_folder(current_user.username) + files = [f for f in os.listdir(folder) if f.endswith((".pdf", ".docx", ".txt", ".md"))] + return jsonify({"files": files}), 200 + except Exception as e: + return jsonify({"error": str(e)}), 500 + +@app.route("/upload", methods=["POST"]) +@login_required +def upload(): + try: + if not user_has_rag_keys(current_user): + return jsonify({"error": "⚠️ Please add your Gemini API key, Pinecone API key, and Pinecone index name in the Profile page to upload and chat."}), 400 + + if "pdf" not in request.files: + return jsonify({"error": "No file found"}), 400 + + file = request.files["pdf"] + + if file.filename == "": + return jsonify({"error": "No file selected"}), 400 + + if not allowed_file(file.filename): + return jsonify({"error": "Only PDF, DOCX, TXT & MD files allowed"}), 400 + + folder = get_user_upload_folder(current_user.username) + filepath = os.path.join(folder, file.filename) + file.save(filepath) + + chunks = load_and_chunk(filepath) + store_embeddings(chunks, file.filename, current_user) + + return jsonify({"message": f"{file.filename} uploaded successfully!"}), 200 + + except Exception as e: + return jsonify({"error": str(e)}), 500 + +@app.route("/ask", methods=["POST"]) +@login_required +def ask(): + try: + if not user_has_rag_keys(current_user): + return jsonify({"error": "⚠️ Please add your Gemini API key, Pinecone API key, and Pinecone index name in the Profile page to upload and chat."}), 400 + + data = request.get_json() + question = data.get("question", "").strip() + filename = data.get("filename", "").strip() + + if not question: + return jsonify({"error": "Question cannot be empty"}), 400 + + context_chunks = retrieve_chunks(question, filename, current_user) + answer = generate_answer(question, context_chunks, current_user) + + username = current_user.username + if username not in chat_history: + chat_history[username] = [] + + chat_history[username].append({ + "question": question, + "answer": answer + }) + + return jsonify({ + "answer": answer, + "sources": context_chunks + }), 200 + + except Exception as e: + return jsonify({"error": str(e)}), 500 + +@app.route("/history", methods=["GET"]) +@login_required +def history(): + try: + username = current_user.username + return jsonify({"history": chat_history.get(username, [])}), 200 + except Exception as e: + return jsonify({"error": str(e)}), 500 + +@app.route("/clear", methods=["POST"]) +@login_required +def clear(): + try: + username = current_user.username + chat_history[username] = [] + return jsonify({"message": "Chat history cleared!"}), 200 + except Exception as e: + return jsonify({"error": str(e)}), 500 + +@app.route("/delete", methods=["POST"]) +@login_required +def delete(): + try: + data = request.get_json() + filename = data.get("filename", "") + + if not filename: + return jsonify({"error": "Filename not provided"}), 400 + + folder = get_user_upload_folder(current_user.username) + filepath = os.path.join(folder, filename) + + if not os.path.exists(filepath): + return jsonify({"error": "File not found"}), 404 + + os.remove(filepath) + + # Delete vectors from Pinecone + delete_embeddings(filename, current_user) + + return jsonify({"message": f"{filename} deleted successfully!"}), 200 + + except Exception as e: + return jsonify({"error": str(e)}), 500 + +@app.route("/clear_vectorstore", methods=["POST"]) +@login_required +def clear_vectorstore(): + try: + # Clear all vectors in user's Pinecone namespace + clear_all_embeddings(current_user) + return jsonify({"message": "Vector store cleared successfully!"}), 200 + + except Exception as e: + return jsonify({"error": str(e)}), 500 + +# ── Run ─────────────────────────────────────────────── +if __name__ == "__main__": + app.run(debug=True, host="0.0.0.0", port=5000) \ No newline at end of file diff --git a/backend/app/audit.py b/backend/app/audit.py deleted file mode 100644 index f1ca3ad7..00000000 --- a/backend/app/audit.py +++ /dev/null @@ -1,42 +0,0 @@ -from __future__ import annotations - -from typing import Any, Optional -from loguru import logger - - -def audit_log( - *, - action: str, - user_id: Optional[str] = None, - result: str = "success", - ip_address: Optional[str] = None, - resource: Optional[str] = None, - details: Optional[dict[str, Any]] = None, -) -> None: - """Emit a structured audit log entry. - - Args: - action: Dot-separated action identifier, e.g. ``user.login``, - ``api_key.create``, ``admin.suspend_user``. - user_id: ID of the acting user (may be None for unauthenticated - actions like failed logins). - result: ``"success"`` or ``"failure"``. - ip_address: Client IP address from the request. - resource: Affected resource identifier (e.g. API key ID, - document ID, target user ID). - details: Any additional context (method, reason, etc.). - """ - logger.bind( - audit=True, - action=action, - actor_id=user_id or "anonymous", - result=result, - ip_address=ip_address or "unknown", - resource=resource or "", - audit_details=details or {}, - ).info( - "AUDIT | {action} | user={actor} | result={result}", - action=action, - actor=user_id or "anonymous", - result=result, - ) diff --git a/backend/app/auth.py b/backend/app/auth.py index dae5bf37..53db8780 100644 --- a/backend/app/auth.py +++ b/backend/app/auth.py @@ -6,23 +6,17 @@ import jwt import bcrypt -from fastapi import Depends, HTTPException, status, Cookie, Request +from fastapi import Depends, HTTPException, status, Cookie from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials from sqlalchemy.orm import Session from app.config import get_settings from app.database import get_db -from app.exceptions import ForbiddenException from app.models import User, UserRole settings = get_settings() security = HTTPBearer(auto_error=False) -# Whitelist of allowed JWT signing algorithms. -# Only HMAC-SHA256 is permitted; asymmetric / experimental algorithms are -# explicitly excluded to reduce the attack surface. -ALLOWED_JWT_ALGORITHMS: set[str] = {"HS256"} - # ── Password Hashing ───────────────────────────────── @@ -60,8 +54,6 @@ def create_refresh_token(user_id) -> str: def decode_token(token: str, token_type: str = "access") -> Optional[str]: """Decode JWT and return user_id, or None if invalid.""" - if settings.JWT_ALGORITHM not in ALLOWED_JWT_ALGORITHMS: - raise ValueError(f"JWT algorithm {settings.JWT_ALGORITHM} is not in the allowed whitelist") try: payload = jwt.decode(token, settings.SECRET_KEY, algorithms=[settings.JWT_ALGORITHM]) if payload.get("type") != token_type: @@ -88,8 +80,6 @@ def create_invite_token(inviter_id: str, email: str, workspace_name: str) -> str def decode_invite_token(token: str) -> Optional[dict[str, Any]]: """Decode a workspace invite JWT and return its payload if valid.""" - if settings.JWT_ALGORITHM not in ALLOWED_JWT_ALGORITHMS: - raise ValueError(f"JWT algorithm {settings.JWT_ALGORITHM} is not in the allowed whitelist") try: payload = jwt.decode(token, settings.SECRET_KEY, algorithms=[settings.JWT_ALGORITHM]) if payload.get("type") != "invite": @@ -106,7 +96,6 @@ def decode_invite_token(token: str) -> Optional[dict[str, Any]]: import hashlib def get_current_user( - request: Request, credentials: Optional[HTTPAuthorizationCredentials] = Depends(security), access_token: Optional[str] = Cookie(None), db: Session = Depends(get_db), @@ -146,10 +135,6 @@ def get_current_user( status_code=status.HTTP_401_UNAUTHORIZED, detail="User not found for this API key", ) - # Store user ID on request state and set context variable - request.state.user_id = user.id - from app.observability import user_id_var - user_id_var.set(user.id) return user # Otherwise, process as JWT @@ -170,11 +155,6 @@ def get_current_user( detail="User not found", ) - # Store user ID on request state and set context variable - request.state.user_id = user.id - from app.observability import user_id_var - user_id_var.set(user.id) - return user @@ -199,7 +179,10 @@ def get_admin_user(user: User = Depends(get_current_user)) -> User: Raises 403 Forbidden if the user lacks sufficient permissions. """ if not _is_admin_user(user): - raise ForbiddenException("Admin access required") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Admin access required", + ) return user diff --git a/backend/app/cache.py b/backend/app/cache.py deleted file mode 100644 index 07a83505..00000000 --- a/backend/app/cache.py +++ /dev/null @@ -1,178 +0,0 @@ -""" -Response caching utility for PDF-Assistant-RAG. - -Supports two backends: -- Redis (preferred, for production) -- LRU in-memory cache (fallback for development or when Redis is unavailable) - -Cache key is a SHA-256 hash of (user_id, document_id, question) to ensure -keys are short, stable, and unique across all user/question/document -combinations — never shared between different users. -""" - -import hashlib -import json -import logging -import os -from typing import Optional - -logger = logging.getLogger(__name__) - -# --------------------------------------------------------------------------- -# Configuration — all values come from environment variables -# --------------------------------------------------------------------------- - -CACHE_TTL: int = int(os.getenv("CACHE_TTL", "3600")) # seconds; default 1 hour -REDIS_URL: Optional[str] = os.getenv("REDIS_URL", None) -LRU_MAX_SIZE: int = int(os.getenv("CACHE_LRU_MAX_SIZE", "128")) - -# --------------------------------------------------------------------------- -# Redis client (lazy init — only created when REDIS_URL is set) -# --------------------------------------------------------------------------- - -_redis_client = None -_redis_available = False - - -def _get_redis(): - """ - Lazily initialise the Redis client. - Returns the client if Redis is reachable, otherwise returns None. - After one failure, stops retrying for the process lifetime. - """ - global _redis_client, _redis_available - - if _redis_client is not None: - return _redis_client if _redis_available else None - - if not REDIS_URL: - logger.info("REDIS_URL not set — using LRU in-memory cache.") - _redis_available = False - return None - - try: - import redis # soft import — redis package is optional - - client = redis.from_url( - REDIS_URL, - decode_responses=True, - socket_connect_timeout=2, - ) - client.ping() - _redis_client = client - _redis_available = True - logger.info("Redis cache connected at %s", REDIS_URL) - return _redis_client - except Exception as exc: # noqa: BLE001 - logger.warning("Redis unavailable (%s) — falling back to LRU cache.", exc) - _redis_available = False - return None - - -# --------------------------------------------------------------------------- -# LRU in-memory fallback -# --------------------------------------------------------------------------- - -_lru_store: dict = {} -_lru_order: list = [] - - -def _lru_get(key: str) -> Optional[str]: - return _lru_store.get(key) - - -def _lru_set(key: str, value: str) -> None: - if key in _lru_store: - _lru_order.remove(key) - elif len(_lru_store) >= LRU_MAX_SIZE: - oldest = _lru_order.pop(0) - del _lru_store[oldest] - _lru_store[key] = value - _lru_order.append(key) - - -def _lru_delete(key: str) -> None: - if key in _lru_store: - del _lru_store[key] - _lru_order.remove(key) - - -# --------------------------------------------------------------------------- -# Public API — these are the only functions chat.py needs -# --------------------------------------------------------------------------- - - -def make_cache_key(user_id: str, document_id: str, question: str) -> str: - """ - Generate a stable, short cache key from user_id + document_id + question. - - SHA-256 gives us a 64-char hex string that is: - - Always the same length regardless of question length - - Unique per (user_id, document_id, question) triple — user_id is - required so two different users asking the identical question never - collide on the same cache entry, even when document_id is empty - (cross-document queries against a user's own private knowledge base) - - Safe for Redis keys and dict keys - """ - raw = f"{user_id}:{document_id}:{question.strip().lower()}" - return hashlib.sha256(raw.encode("utf-8")).hexdigest() - - -def get_cached_response(user_id: str, document_id: str, question: str) -> Optional[str]: - """ - Look up a cached answer for a (user_id, document_id, question) triple. - Returns the answer string on hit, None on miss. - """ - key = make_cache_key(user_id, document_id, question) - r = _get_redis() - - if r is not None: - try: - value = r.get(key) - if value: - logger.debug("Cache HIT (Redis) for key %s", key[:12]) - return json.loads(value) - except Exception as exc: # noqa: BLE001 - logger.warning("Redis GET failed (%s) — checking LRU.", exc) - - value = _lru_get(key) - if value: - logger.debug("Cache HIT (LRU) for key %s", key[:12]) - return json.loads(value) - - logger.debug("Cache MISS for key %s", key[:12]) - return None - - - -def set_cached_response(user_id: str, document_id: str, question: str, answer: str) -> None: - """ - Store an answer. Tries Redis first; falls back to LRU. - TTL is controlled by the CACHE_TTL environment variable. - """ - key = make_cache_key(user_id, document_id, question) - serialised = json.dumps(answer) - r = _get_redis() - - if r is not None: - try: - r.setex(key, CACHE_TTL, serialised) - logger.debug("Cache SET (Redis) key %s TTL %ds", key[:12], CACHE_TTL) - return - except Exception as exc: # noqa: BLE001 - logger.warning("Redis SET failed (%s) — storing in LRU.", exc) - - _lru_set(key, serialised) - logger.debug("Cache SET (LRU) key %s", key[:12]) - - -def invalidate_cache(user_id: str, document_id: str, question: str) -> None: - """Remove one cache entry — useful when a document is re-indexed.""" - key = make_cache_key(user_id, document_id, question) - r = _get_redis() - if r is not None: - try: - r.delete(key) - except Exception: # noqa: BLE001 - pass - _lru_delete(key) diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py index 25ec3edf..4cfe44d1 100644 --- a/backend/app/celery_app.py +++ b/backend/app/celery_app.py @@ -1,19 +1,23 @@ -import os +"""Celery application configured for Redis-backed background jobs.""" from celery import Celery -# Initialize the Celery application instance +from app.config import get_settings + + +settings = get_settings() + celery_app = Celery( - "worker", - broker=os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0"), - backend=os.getenv("CELERY_RESULT_BACKEND", "redis://localhost:6379/0") + "pdf_assistant_rag", + broker=settings.CELERY_BROKER_URL, + backend=settings.CELERY_RESULT_BACKEND, + include=["app.tasks"], ) -# Optional configuration updates for reliable serialization celery_app.conf.update( + task_track_started=settings.CELERY_TASK_TRACK_STARTED, task_serializer="json", - accept_content=["json"], result_serializer="json", + accept_content=["json"], + timezone="UTC", ) -# Tell Celery to discover background tasks dynamically to break circular loops -celery_app.autodiscover_tasks(["app"]) \ No newline at end of file diff --git a/backend/app/config.py b/backend/app/config.py index c517b147..23563d07 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -5,7 +5,7 @@ import os from pydantic_settings import BaseSettings from functools import lru_cache -from pydantic import model_validator + class Settings(BaseSettings): # ── App ────────────────────────────────────────────── @@ -15,16 +15,8 @@ class Settings(BaseSettings): ENVIRONMENT: str = "development" ALLOWED_ORIGINS: str = "http://localhost:3000,http://localhost:7860" - # ── Logging ────────────────────────────────────────── - LOG_LEVEL: str | None = None - LOG_FILE: str = "./data/logs/app.log" - - # ── Database ───────────────────────────────────────── DATABASE_URL: str = "sqlite:///./data/app.db" - DATABASE_POOL_SIZE: int = 10 - DATABASE_MAX_OVERFLOW: int = 20 - DATABASE_POOL_PRE_PING: bool = True # ── Auth ───────────────────────────────────────────── JWT_ALGORITHM: str = "HS256" @@ -58,12 +50,6 @@ class Settings(BaseSettings): CELERY_RESULT_BACKEND: str = "redis://localhost:6379/1" CELERY_TASK_TRACK_STARTED: bool = True - # ── Document Processing ────────────────────────────── - DOC_PROCESSING_TIMEOUT_MINUTES: int = 30 - DOC_PROCESSING_MAX_RETRIES: int = 3 - DOC_PROCESSING_RETRY_DELAY_SECONDS: int = 30 - DOC_CLEANUP_MAX_AGE_DAYS: int = 90 - # ── File Upload ────────────────────────────────────── UPLOAD_DIR: str = "./data/uploads" MAX_UPLOAD_SIZE_MB: int = 50 @@ -85,10 +71,6 @@ class Settings(BaseSettings): TOP_K_RETRIEVAL: int = 20 # Fetch more candidates for reranking TOP_K_RERANK: int = 8 # Final number of chunks to return after reranking - # ── Hybrid Search / RRF ─────────────────────────────── - USE_HYBRID_SEARCH: bool = True # set to False to fall back to vector-only - RRF_K: int = 60 # RRF rank constant; 60 is the standard default - # ── Knowledge Graph (GraphRAG) ─────────────────────── GRAPH_PERSIST_DIR: str = "./data/graphs" GRAPH_ENTITY_LABELS: set = { @@ -119,17 +101,6 @@ class Settings(BaseSettings): LLM_TEMPERATURE: float = 0.3 SUMMARY_MAX_TOKENS: int = 512 - # ── Field-level Encryption ──────────────────────── - # Dedicated key for encrypting sensitive user fields (tokens, secrets). - # Must be overridden in production — validate_production() enforces this. - # Generate a strong key: python -c "import secrets; print(secrets.token_urlsafe(32))" - FIELD_ENCRYPTION_KEY: str = "change-me-in-production-field-encryption-key" - FIELD_ENCRYPTION_KEY_VERSION: int = 1 - - # ── Document Cleanup ───────────────────────────── - DOC_CLEANUP_ENABLED: bool = True - DOC_CLEANUP_INACTIVE_DAYS: int = 30 - # ── LangSmith Tracing (optional) ───────────────────── LANGSMITH_TRACING: bool = False LANGSMITH_API_KEY: str = "" @@ -152,75 +123,11 @@ class Settings(BaseSettings): SMTP_USER: str = "" SMTP_PASSWORD: str = "" - # ── Database Tuning ───────────────────────────────── - DATABASE_POOL_RECYCLE: int = 3600 - DATABASE_SLOW_QUERY_THRESHOLD: float = 1.0 - - # ── Request Limits ─────────────────────────────────── - MAX_REQUEST_BODY_SIZE_MB: int = 50 - @property def cors_origins(self) -> list[str]: - origins = [ - "http://localhost:3000", - "http://127.0.0.1:3000", - "https://pdf-assistant-rag.vercel.app", - ] - if self.ALLOWED_ORIGINS: - for o in self.ALLOWED_ORIGINS.split(","): - o_strip = o.strip() - if o_strip and o_strip not in origins: - origins.append(o_strip) - return origins - - @model_validator(mode="after") - def validate_vision_provider_keys(self) -> "Settings": - provider = self.VISION_PROVIDER.lower() if self.VISION_PROVIDER else None - if provider == "openai" and not self.OPENAI_API_KEY: - raise ValueError( - "ValidationError: OPENAI_API_KEY is required when VISION_PROVIDER is set to 'openai'." - ) - return self - - def validate_production(self) -> None: - """Validate that critical secrets are overridden in production. - - Called during application startup (see ``main.py`` lifespan). In - non-production environments it only logs warnings; in production it - raises ``ValueError`` to prevent the app from starting with insecure - defaults. - """ - import logging - _logger = logging.getLogger(__name__) - - _INSECURE_DEFAULTS = { - "SECRET_KEY": "change-me-in-production-please", - "FIELD_ENCRYPTION_KEY": "change-me-in-production-field-encryption-key", - } - - issues: list[str] = [] - for field, default_value in _INSECURE_DEFAULTS.items(): - current = getattr(self, field, None) - if current == default_value: - issues.append( - f"{field} is set to the insecure default. " - f"Override it via environment variable or .env file." - ) - - # Minimum key length check - if len(self.SECRET_KEY) < 32: - issues.append( - "SECRET_KEY is too short — use at least 32 characters. " - "Generate one with: python -c \"import secrets; print(secrets.token_urlsafe(32))\"" - ) - - if self.ENVIRONMENT == "production" and issues: - raise ValueError( - "Production configuration errors:\n • " + "\n • ".join(issues) - ) - elif issues: - for issue in issues: - _logger.warning("Non-production config warning: %s", issue) + if self.ENVIRONMENT == "production": + return [o.strip() for o in self.ALLOWED_ORIGINS.split(",")] + return ["*"] class Config: env_file = ".env" diff --git a/backend/app/database.py b/backend/app/database.py index 02f8890c..b7fa2bd2 100644 --- a/backend/app/database.py +++ b/backend/app/database.py @@ -3,10 +3,8 @@ Uses synchronous SQLAlchemy for simplicity and compatibility. """ import os -import time import logging -from contextlib import contextmanager -from sqlalchemy import create_engine, event, inspect, text +from sqlalchemy import create_engine, inspect, text from sqlalchemy.orm import sessionmaker, declarative_base from app.config import get_settings @@ -31,10 +29,6 @@ engine = create_engine( settings.DATABASE_URL, echo=settings.DEBUG, - pool_size=settings.DATABASE_POOL_SIZE, - max_overflow=settings.DATABASE_MAX_OVERFLOW, - pool_pre_ping=settings.DATABASE_POOL_PRE_PING, - pool_recycle=settings.DATABASE_POOL_RECYCLE, # Recycle stale connections ) SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) @@ -52,76 +46,6 @@ def get_db(): db.close() -@contextmanager -def get_db_session(): - """Context manager for background tasks, streaming, and other uses outside FastAPI DI. - - Creates a new session, commits on success, rolls back SQLAlchemy errors - (converting them to typed AppException), re-raises non-DB exceptions, - and always closes the session. - """ - from sqlalchemy.exc import SQLAlchemyError - from app.exceptions import AppException - - session = SessionLocal() - try: - yield session - session.commit() - except SQLAlchemyError as e: - session.rollback() - raise AppException( - "DATABASE_ERROR", - "A database error occurred while processing your request.", - 500, - {"error": str(e)[:200]}, - ) from e - except Exception: - session.rollback() - raise - finally: - session.close() - -_SLOW_QUERY_THRESHOLD = settings.DATABASE_SLOW_QUERY_THRESHOLD - - -@event.listens_for(engine, "before_cursor_execute") -def _before_cursor_execute(conn, cursor, statement, parameters, context, executemany): - conn.info.setdefault("query_start_time", []).append(time.perf_counter()) - - -@event.listens_for(engine, "after_cursor_execute") -def _after_cursor_execute(conn, cursor, statement, parameters, context, executemany): - start_times = conn.info.get("query_start_time") - if not start_times: - return - total_time = time.perf_counter() - start_times.pop(-1) - if total_time > _SLOW_QUERY_THRESHOLD: - logger.warning( - "Slow query detected (%.2fs): %s", - total_time, - statement[:500], - ) - - -# ── Session Lifecycle Logging (DEBUG only) ─────────── -if settings.DEBUG: - @event.listens_for(SessionLocal, "after_begin") - def _receive_after_begin(session, transaction, connection): - logger.debug("Session %s began transaction", id(session)) - - @event.listens_for(SessionLocal, "after_commit") - def _receive_after_commit(session): - logger.debug("Session %s committed", id(session)) - - @event.listens_for(SessionLocal, "after_rollback") - def _receive_after_rollback(session): - logger.debug("Session %s rolled back", id(session)) - - @event.listens_for(SessionLocal, "after_close") - def _receive_after_close(session): - logger.debug("Session %s closed", id(session)) - - def _migrate_schema(): """Apply schema migrations for existing databases (SQLite-compatible). @@ -144,8 +68,6 @@ def _migrate_schema(): "verification_token_created_at", "ALTER TABLE users ADD COLUMN verification_token_created_at TIMESTAMP", ), - ("users", "display_name", "ALTER TABLE users ADD COLUMN display_name VARCHAR(120)"), - ("users", "avatar_url", "ALTER TABLE users ADD COLUMN avatar_url VARCHAR(500)"), ] for table, column, ddl in users_migrations: if column not in existing_users_columns: @@ -191,14 +113,6 @@ def _migrate_schema(): ("documents", "drive_file_id", "ALTER TABLE documents ADD COLUMN drive_file_id VARCHAR(255)"), ("documents", "drive_folder_id", "ALTER TABLE documents ADD COLUMN drive_folder_id VARCHAR(255)"), ("documents", "drive_synced_at", "ALTER TABLE documents ADD COLUMN drive_synced_at TIMESTAMP"), - ("documents", "processing_progress", "ALTER TABLE documents ADD COLUMN processing_progress INTEGER DEFAULT 0"), - ("documents", "processing_stage", "ALTER TABLE documents ADD COLUMN processing_stage VARCHAR(20) DEFAULT 'queued'"), - ("documents", "retry_count", "ALTER TABLE documents ADD COLUMN retry_count INTEGER DEFAULT 0"), - ("documents", "last_error_traceback", "ALTER TABLE documents ADD COLUMN last_error_traceback TEXT"), - ("documents", "processing_started_at", "ALTER TABLE documents ADD COLUMN processing_started_at TIMESTAMP"), - ("documents", "completed_at", "ALTER TABLE documents ADD COLUMN completed_at TIMESTAMP"), - ("documents", "extracted_urls", "ALTER TABLE documents ADD COLUMN extracted_urls TEXT"), - ("documents", "keywords", "ALTER TABLE documents ADD COLUMN keywords TEXT"), ] for table, column, ddl in docs_migrations: if column not in existing_docs_columns: @@ -218,7 +132,6 @@ def _migrate_schema(): existing_chat_columns = set() chat_migrations = [ ("chat_messages", "feedback", "ALTER TABLE chat_messages ADD COLUMN feedback VARCHAR(10)"), - ("chat_messages", "session_id", "ALTER TABLE chat_messages ADD COLUMN session_id CHAR(36)"), ] for table, column, ddl in chat_migrations: if column not in existing_chat_columns: @@ -230,153 +143,7 @@ def _migrate_schema(): logger.warning( "Migration skipped (may already exist): %s.%s", table, column ) - - # Ensure index exists on session_id - try: - with engine.begin() as conn: - conn.execute(text("CREATE INDEX IF NOT EXISTS ix_chat_messages_session_id ON chat_messages (session_id)")) - except Exception: - pass - - - # Migrate documents — embedding cache tracking - try: - existing_docs_columns = {c["name"] for c in inspector.get_columns("documents")} - except Exception: - existing_docs_columns = set() - - embedding_cache_migrations = [ - ( - "documents", - "extracted_urls", - "ALTER TABLE documents ADD COLUMN extracted_urls TEXT", - ), - ] - for table, column, ddl in embedding_cache_migrations: - if column not in existing_docs_columns: - try: - with engine.begin() as conn: - conn.execute(text(ddl)) - logger.info("Migration: added column %s.%s", table, column) - except Exception: - logger.warning( - "Migration skipped (may already exist): %s.%s", table, column - ) - - - # ── Workspace tables ────────────────────────────────────────────────── - existing_tables = set(inspector.get_table_names()) - - if "workspaces" not in existing_tables: - try: - with engine.begin() as conn: - conn.execute(text(""" - CREATE TABLE workspaces ( - id CHAR(36) PRIMARY KEY, - name VARCHAR(255) NOT NULL, - created_by CHAR(36) NOT NULL REFERENCES users(id), - created_at TIMESTAMP - ) - """)) - conn.execute(text( - "CREATE INDEX IF NOT EXISTS ix_workspaces_created_by " - "ON workspaces (created_by)" - )) - logger.info("Migration: created table workspaces") - except Exception: - logger.warning("Migration skipped (may already exist): workspaces") - - if "workspace_members" not in existing_tables: - try: - with engine.begin() as conn: - conn.execute(text(""" - CREATE TABLE workspace_members ( - id CHAR(36) PRIMARY KEY, - workspace_id CHAR(36) NOT NULL REFERENCES workspaces(id), - user_id CHAR(36) NOT NULL REFERENCES users(id), - role VARCHAR(20) NOT NULL DEFAULT 'viewer', - joined_at TIMESTAMP, - CONSTRAINT uq_workspace_member UNIQUE (workspace_id, user_id) - ) - """)) - conn.execute(text( - "CREATE INDEX IF NOT EXISTS ix_workspace_members_workspace_id " - "ON workspace_members (workspace_id)" - )) - conn.execute(text( - "CREATE INDEX IF NOT EXISTS ix_workspace_members_user_id " - "ON workspace_members (user_id)" - )) - logger.info("Migration: created table workspace_members") - except Exception: - logger.warning("Migration skipped (may already exist): workspace_members") - - if "workspace_invitations" not in existing_tables: - try: - with engine.begin() as conn: - conn.execute(text(""" - CREATE TABLE workspace_invitations ( - id CHAR(36) PRIMARY KEY, - email VARCHAR(120) NOT NULL, - token_hash VARCHAR(255) NOT NULL UNIQUE, - inviter_id CHAR(36) NOT NULL REFERENCES users(id), - workspace_name VARCHAR(255) NOT NULL, - created_at TIMESTAMP, - expires_at TIMESTAMP NOT NULL, - accepted_at TIMESTAMP - ) - """)) - conn.execute(text( - "CREATE INDEX IF NOT EXISTS ix_workspace_invitations_email " - "ON workspace_invitations (email)" - )) - conn.execute(text( - "CREATE INDEX IF NOT EXISTS ix_workspace_invitations_token_hash " - "ON workspace_invitations (token_hash)" - )) - conn.execute(text( - "CREATE INDEX IF NOT EXISTS ix_workspace_invitations_inviter_id " - "ON workspace_invitations (inviter_id)" - )) - logger.info("Migration: created table workspace_invitations") - except Exception: - logger.warning("Migration skipped (may already exist): workspace_invitations") - - -def advisory_lock(lock_id: int): - """Context manager that acquires a PostgreSQL advisory lock (xact scope). - - On SQLite the lock is a no-op because SQLite serializes all writes anyway. - On PostgreSQL the lock is released automatically at transaction commit. - - Usage:: - - with advisory_lock(hash("cleanup_inactive") & 0x7FFFFFFF): - ... - """ - if is_sqlite: - # SQLite serializes writes; no explicit lock needed. - return _noop_contextmanager() - - from contextlib import contextmanager - - @contextmanager - def _pg_lock(): - with engine.begin() as conn: - conn.execute(text("SELECT pg_advisory_xact_lock(:id)"), {"id": lock_id}) - yield - - return _pg_lock() - - -def _noop_contextmanager(): - from contextlib import contextmanager as _cm - - @_cm - def _noop(): - yield - return _noop() def init_db(): diff --git a/backend/app/email_service.py b/backend/app/email_service.py index 6fcb858a..47d1bd5f 100644 --- a/backend/app/email_service.py +++ b/backend/app/email_service.py @@ -40,156 +40,3 @@ def send_email(to: str, subject: str, body: str, html: str | None = None) -> Non subject, body, ) - - -def send_workspace_invite_email( - to: str, - workspace_name: str, - invite_link: str, - expires_in_hours: int, - personal_message: str | None = None, -) -> None: - """Send a workspace invitation email with an HTML body. - - Builds a branded HTML email containing the workspace name, an optional - personal message from the inviter, a prominent call-to-action button with - the acceptance link, and an expiry notice. Falls back to plain-text if HTML - is not supported by the client. Delegates delivery to :func:`send_email`. - - Args: - to: Recipient email address. - workspace_name: Name of the workspace the recipient is invited to. - invite_link: Fully-qualified URL the recipient must visit to accept. - expires_in_hours: How many hours until the invite token expires. - personal_message: Optional personal note from the inviting admin. - """ - subject = f"You're invited to join workspace '{workspace_name}'" - - # ── Plain-text fallback ─────────────────────────────────────────────── - plain_lines = [ - "Hello,", - "", - f"You have been invited to join the workspace '{workspace_name}'.", - ] - if personal_message: - plain_lines += ["", personal_message] - plain_lines += [ - "", - "Accept your invitation by visiting the link below:", - invite_link, - "", - f"This invitation expires in {expires_in_hours} hours.", - "", - "If you did not expect this email, you can safely ignore it.", - ] - plain_body = "\n".join(plain_lines) - - # ── HTML body ───────────────────────────────────────────────────────── - personal_block = "" - if personal_message: - personal_block = f""" - - - """ - - html_body = f""" - - - - - {subject} - - -
### 👤 Users -- 🔐 JWT-secured register, login & email verification -- 📄 Upload **PDF**, **DOCX**, **TXT**, and **Markdown** -- 🌐 URL ingestion via web crawler +- 🔐 JWT-secured register & login +- 📄 Upload **PDF** and **DOCX** documents - 💬 Ask questions in natural language - 🌊 **Streaming AI responses** token-by-token - 📚 Inline **source citations** with page numbers -- 📥 Export chat as **Markdown, TXT, or PDF** - 🗂️ Per-user complete data isolation ### ⚙️ Engineering -- 🚀 **Async FastAPI** with SSE streaming -- 🗄️ **PostgreSQL** metadata + **ChromaDB** vectors -- 🔄 **Celery + Redis** async ingestion pipeline -- 🐳 **Multi-stage Docker** with CPU & GPU profiles -- 📊 **Prometheus metrics** + Grafana dashboard -- 🩺 **Deep health endpoint** — DB, Redis, Celery, ChromaDB -- 🔒 Rate limiting, CORS, file validation, JWT expiry -- 🧪 **Playwright** E2E + visual regression tests +- 🚀 **Async FastAPI** with Server-Sent Events streaming +- 🗄️ **ChromaDB** with persistent per-user collections +- 🐳 **Multi-stage Docker** build (Node → Python) +- 🔄 **GitHub Actions CI** on `dev` branch +- 🛡️ CORS, file validation, JWT expiry +- 📊 Chat **history persistence** per document
-

- {personal_message} -

-
- - - -
- - - - - - - - - - - - - {personal_block} - - - - - - - - - - - - - - - - -
-

- 📄 PDF Assistant -

-

- Workspace Invitation -

-
-

- You've been invited! -

-

- You have been invited to join the workspace - '{workspace_name}' - on PDF Assistant. Accept below to start collaborating. -

-
- - Accept Invitation → - -

- Or copy this link into your browser:
- {invite_link} -

-
-

- ⏳ This invitation expires in {expires_in_hours} hours. -

-
-

- If you did not expect this invitation, you can safely ignore this email.
- This email was sent by PDF Assistant · No reply -

-
-
- -""" - - send_email(to, subject, plain_body, html=html_body) - logger.info( - "Workspace invite email dispatched to %s for workspace '%s'", - to, - workspace_name, - ) diff --git a/backend/app/exceptions.py b/backend/app/exceptions.py deleted file mode 100644 index 74a33703..00000000 --- a/backend/app/exceptions.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Custom exception hierarchy for standardized error handling.""" - - -class AppException(Exception): - """Base exception for all application-level errors.""" - - def __init__(self, code: str, message: str, status_code: int = 500, details: dict = None): - self.code = code - self.message = message - self.status_code = status_code - self.details = details or {} - super().__init__(self.message) - - -class NotFoundException(AppException): - def __init__(self, resource: str, identifier: str = None): - msg = f"{resource.title()} not found" - if identifier: - msg = f"{resource.title()} '{identifier}' not found" - super().__init__( - f"{resource.upper()}_NOT_FOUND", - msg, - 404, - {resource: identifier} if identifier else {}, - ) - - -class UnauthorizedException(AppException): - def __init__(self, message: str = "Authentication required"): - super().__init__("UNAUTHORIZED", message, 401) - - -class ForbiddenException(AppException): - def __init__(self, message: str = "You do not have permission to perform this action"): - super().__init__("FORBIDDEN", message, 403) - - -class ConflictException(AppException): - def __init__(self, message: str, details: dict = None): - super().__init__("CONFLICT", message, 409, details or {}) - - -class ValidationException(AppException): - def __init__(self, message: str, details: dict = None): - super().__init__("VALIDATION_ERROR", message, 400, details or {}) - - -class RateLimitException(AppException): - def __init__(self, message: str = "Rate limit exceeded. Please try again later."): - super().__init__("RATE_LIMIT_EXCEEDED", message, 429) - - -class ExternalServiceException(AppException): - def __init__(self, service: str, message: str = None): - msg = message or f"External service '{service}' returned an error" - super().__init__(f"{service.upper()}_ERROR", msg, 502, {"service": service}) - - -class UnsafePromptException(AppException): - def __init__(self, message: str = None): - msg = message or "Your message contains prohibited content and was blocked." - super().__init__("UNSAFE_PROMPT", msg, 400) diff --git a/backend/app/main.py b/backend/app/main.py index 97d8b3a1..b5e6445f 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -3,56 +3,101 @@ Mounts all routes, configures CORS, and serves the Next.js frontend build. """ import os -import uuid -import signal -import asyncio import logging from contextlib import asynccontextmanager from fastapi import FastAPI, Request from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware -from fastapi.middleware.trustedhost import TrustedHostMiddleware -from fastapi.responses import JSONResponse -from starlette.middleware.base import BaseHTTPMiddleware +from fastapi.staticfiles import StaticFiles +from fastapi.responses import FileResponse, JSONResponse from sqlalchemy import select from sqlalchemy.exc import SQLAlchemyError -from app.security import verify_secure_sandbox_path + from slowapi.errors import RateLimitExceeded from slowapi.middleware import SlowAPIMiddleware from app.config import get_settings -from app.exceptions import AppException from app.rate_limit import limiter from app.database import init_db, get_db -from app.observability import setup_prometheus_metrics, setup_logging, StructuredLoggingMiddleware +from app.observability import setup_prometheus_metrics from app.rag.vectorstore import get_chroma_client from app.scheduler import start_scheduler, stop_scheduler from app.routes.profile import router as profile_router -from app.routes.health import router as health_router - -# Configure logging using loguru structured JSON logging -setup_logging() -from loguru import logger +# Configure logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", +) +logger = logging.getLogger(__name__) settings = get_settings() +async def document_cleanup_job(): + """Background loop to periodically purge documents not accessed in 30 days.""" + import asyncio + from datetime import datetime, timedelta, timezone + logger.info("Starting document cleanup background job loop") + while True: + try: + from app.database import SessionLocal + from app.models import Document + from app.rag.vectorstore import delete_document_chunks + from sqlalchemy import or_ + + db = SessionLocal() + try: + cutoff = datetime.now(timezone.utc) - timedelta(days=30) + expired_docs = db.query(Document).filter( + or_( + Document.last_accessed_at < cutoff, + Document.last_accessed_at.is_(None) & (Document.uploaded_at < cutoff) + ) + ).all() + + for doc in expired_docs: + logger.info(f"Auto-cleanup: Purging document {doc.id} ('{doc.original_name}') due to inactivity since {doc.last_accessed_at or doc.uploaded_at}") + + # Delete physical file + filepath = os.path.join(settings.UPLOAD_DIR, doc.user_id, doc.filename) + if os.path.exists(filepath): + try: + os.remove(filepath) + except Exception as e: + logger.warning(f"Auto-cleanup: Failed to delete physical file {filepath}: {e}") + + # Delete vectors + try: + delete_document_chunks(document_id=doc.id, user_id=doc.user_id) + except Exception as e: + logger.warning(f"Auto-cleanup: Error deleting vectors for document {doc.id}: {e}") + + # Delete database record + db.delete(doc) + + db.commit() + if expired_docs: + logger.info(f"Auto-cleanup: Purged {len(expired_docs)} documents.") + except Exception as exc: + logger.error(f"Auto-cleanup job encountered error: {exc}", exc_info=True) + finally: + db.close() + + except Exception as e: + logger.error(f"Error in document cleanup background loop: {e}", exc_info=True) + + # Run every 24 hours (86400 seconds) + await asyncio.sleep(86400) + + @asynccontextmanager async def lifespan(app: FastAPI): """Application startup/shutdown lifecycle.""" # ── Startup ────────────────────────────────────── - app.state.is_shutting_down = False logger.info(f"Starting {settings.APP_NAME}") - # Validate production settings - try: - settings.validate_production() - except ValueError as e: - logger.error("Configuration error: %s", e) - raise - # Create tables init_db() logger.info("Database initialized") @@ -69,15 +114,22 @@ async def lifespan(app: FastAPI): except Exception as e: logger.warning(f"Failed to pre-load embedding model: {e}") + # Start background cleanup task + import asyncio + cleanup_task = asyncio.create_task(document_cleanup_job()) + yield - # ── Graceful Shutdown ──────────────────────────── - logger.info("Shutdown signal received — draining in-flight requests") - app.state.is_shutting_down = True - # Give in-flight requests a short window to complete - await asyncio.sleep(5) + # ── Shutdown ───────────────────────────────────── stop_scheduler() - logger.info("Shutdown complete") + logger.info("Shutting down") + cleanup_task.cancel() + try: + await cleanup_task + except asyncio.CancelledError: + pass + except Exception as e: + logger.warning(f"Error cancelling cleanup task: {e}") # ── Create App ─────────────────────────────────────── @@ -89,86 +141,35 @@ async def lifespan(app: FastAPI): ) app.state.limiter = limiter - - -# ── Request ID Middleware ───────────────────────────── -@app.middleware("http") -async def add_request_id(request: Request, call_next): - request_id = str(uuid.uuid4())[:8] - request.state.request_id = request_id - response = await call_next(request) - response.headers["X-Request-ID"] = request_id - return response - - -# ── Global Exception Handlers ───────────────────────── -@app.exception_handler(AppException) -async def app_exception_handler(request: Request, exc: AppException): - return JSONResponse( - status_code=exc.status_code, - content={ - "error": { - "code": exc.code, - "message": exc.message, - "details": exc.details, - "request_id": getattr(request.state, "request_id", None), - } - }, - ) - - -@app.exception_handler(RateLimitExceeded) -async def rate_limit_handler(request: Request, exc: RateLimitExceeded): - return JSONResponse( +app.add_exception_handler( + RateLimitExceeded, + lambda request, exc: JSONResponse( status_code=429, - content={ - "error": { - "code": "RATE_LIMIT_EXCEEDED", - "message": "Rate limit exceeded. Please try again later.", - "details": {}, - "request_id": getattr(request.state, "request_id", None), - } - }, - ) - + content={"detail": "Rate limit exceeded. Please try again later."}, + ), +) @app.exception_handler(RequestValidationError) async def validation_exception_handler(request: Request, exc: RequestValidationError): - details = [ - {"field": " -> ".join(str(p) for p in e.get("loc", [])), "message": e.get("msg", "")} - for e in exc.errors() - ] - return JSONResponse( - status_code=422, - content={ - "error": { - "code": "VALIDATION_ERROR", - "message": "Request validation failed", - "details": {"errors": details}, - "request_id": getattr(request.state, "request_id", None), - } - }, - ) - + def _sanitize_error(err): + if isinstance(err, dict): + sanitized = {} + for k, v in err.items(): + if k == "ctx" and isinstance(v, dict) and "error" in v: + ctx_error = v.get("error") + if isinstance(ctx_error, Exception): + v = {**v, "error": str(ctx_error)} + sanitized[k] = _sanitize_error(v) + return sanitized + if isinstance(err, list): + return [_sanitize_error(i) for i in err] + return err -@app.exception_handler(Exception) -async def unhandled_exception_handler(request: Request, exc: Exception): - logger.exception("Unhandled exception: %s", exc) - if settings.DEBUG: - raise return JSONResponse( - status_code=500, - content={ - "error": { - "code": "INTERNAL_ERROR", - "message": "An unexpected error occurred", - "details": {}, - "request_id": getattr(request.state, "request_id", None), - } - }, + status_code=422, + content={"detail": [_sanitize_error(e) for e in exc.errors()]}, ) - app.add_middleware(SlowAPIMiddleware) # ── CORS (allow frontend dev server) ───────────────── @@ -181,64 +182,6 @@ async def unhandled_exception_handler(request: Request, exc: Exception): ) logger.info(f"CORS origins: {settings.cors_origins}") -# Security Headers Middleware -class SecurityHeadersMiddleware(BaseHTTPMiddleware): - """Add security-critical HTTP headers to every API response.""" - - async def dispatch(self, request: Request, call_next): - response = await call_next(request) - response.headers["X-Content-Type-Options"] = "nosniff" - response.headers["X-Frame-Options"] = "DENY" - response.headers["X-XSS-Protection"] = "1; mode=block" - response.headers["Strict-Transport-Security"] = ( - "max-age=31536000; includeSubDomains" - ) - response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin" - response.headers["Content-Security-Policy"] = ( - "default-src 'self'; " - "script-src 'self' 'wasm-unsafe-eval'; " - "style-src 'self' 'unsafe-inline'; " - "img-src 'self' data: https:; " - "font-src 'self' https:; " - "connect-src 'self' https:;" - ) - response.headers["Permissions-Policy"] = ( - "camera=(), microphone=(), geolocation=()" - ) - return response - -app.add_middleware(SecurityHeadersMiddleware) - - -# Request Body Size Limit Middleware -_MAX_BODY_BYTES = settings.MAX_REQUEST_BODY_SIZE_MB * 1024 * 1024 - - -@app.middleware("http") -async def limit_request_body_size(request: Request, call_next): - content_type = request.headers.get("content-type", "") - if ( - request.method in ("POST", "PUT", "PATCH") - and "multipart/form-data" not in content_type - ): - content_length = request.headers.get("content-length") - if content_length and int(content_length) > _MAX_BODY_BYTES: - return JSONResponse( - status_code=413, - content={ - "error": { - "code": "REQUEST_TOO_LARGE", - "message": f"Request body exceeds the {settings.MAX_REQUEST_BODY_SIZE_MB}MB limit", - "details": {}, - } - }, - ) - return await call_next(request) - - -# Add structured logging middleware as the outermost middleware -app.add_middleware(StructuredLoggingMiddleware) - # ── Mount API Routes ───────────────────────────────── from app.routes.auth import router as auth_router from app.routes.documents import router as documents_router @@ -253,7 +196,6 @@ async def limit_request_body_size(request: Request, call_next): app.include_router(github_router, prefix="/api/v1") app.include_router(admin_router, prefix="/api/v1") app.include_router(workspaces_router, prefix="/api/v1") -app.include_router(health_router, prefix="/api/v1") setup_prometheus_metrics(app) @@ -261,12 +203,6 @@ async def limit_request_body_size(request: Request, call_next): # ── Health Check ───────────────────────────────────── @app.get("/api/health") def health_check(): - # Return 503 during graceful shutdown so load balancers stop routing - if getattr(app.state, "is_shutting_down", False): - return JSONResponse( - status_code=503, - content={"status": "shutting_down", "app": settings.APP_NAME}, - ) return { "status": "healthy", "app": settings.APP_NAME, @@ -296,28 +232,70 @@ def db_health(): except Exception: chroma_status = "down" - if db_status == "up" and chroma_status == "up": - overall_status = "healthy" - elif db_status == "down" and chroma_status == "down": - overall_status = "unhealthy" - else: - overall_status = "degraded" - - return { - "status": overall_status, + overall_status = "ok" if db_status == "up" and chroma_status == "up" else "degraded" + return{ + "status": db_status, "chroma": chroma_status, "db": db_status } -# ── API Root ────────────────────────────────────────── -# Frontend is hosted separately on Vercel/Netlify. -# This backend serves only the API. -@app.get("/") -def root(): - return { - "message": f"Welcome to {settings.APP_NAME} API", - "docs": "/docs", - "health": "/api/health", - } - -app.include_router(profile_router) +# ── Serve Next.js Frontend (production) ────────────── +# In local development, frontend build is at ../../frontend/out relative to backend/app/main.py +# In Docker container (where app is copied to /app/app), frontend build is at /app/frontend/out (which is ../frontend/out relative to /app/app/main.py) +_local_build_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "frontend", "out")) +_docker_build_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "frontend", "out")) + +if os.path.exists(_docker_build_dir): + FRONTEND_BUILD_DIR = _docker_build_dir +else: + FRONTEND_BUILD_DIR = _local_build_dir + +if os.path.exists(FRONTEND_BUILD_DIR): + # Serve static assets (JS, CSS, images) + app.mount("/_next", StaticFiles(directory=os.path.join(FRONTEND_BUILD_DIR, "_next")), name="next_static") + + # Serve other static files if they exist + static_dir = os.path.join(FRONTEND_BUILD_DIR, "static") + if os.path.exists(static_dir): + app.mount("/static", StaticFiles(directory=static_dir), name="static") + + @app.api_route("/{full_path:path}", methods=["GET", "HEAD"]) + async def serve_frontend(full_path: str): + """Serve Next.js static export — tries exact file, then .html, then index.html.""" + # Try exact file path + file_path = os.path.join(FRONTEND_BUILD_DIR, full_path) + if os.path.isfile(file_path): + return FileResponse(file_path) + + # Try with .html extension + html_path = os.path.join(FRONTEND_BUILD_DIR, f"{full_path}.html") + if os.path.isfile(html_path): + return FileResponse(html_path) + + # Try .txt for RSC payloads (Next.js uses .txt for RSC data) + txt_path = os.path.join(FRONTEND_BUILD_DIR, f"{full_path}.txt") + if os.path.isfile(txt_path): + return FileResponse(txt_path) + + # Try as directory index + index_path = os.path.join(FRONTEND_BUILD_DIR, full_path, "index.html") + if os.path.isfile(index_path): + return FileResponse(index_path) + + # Fallback to root index.html (SPA routing) + root_index = os.path.join(FRONTEND_BUILD_DIR, "index.html") + if os.path.isfile(root_index): + return FileResponse(root_index) + + return FileResponse(root_index) if os.path.exists(root_index) else {"error": "Not found"} +else: + logger.info("No frontend build found — running in API-only mode") + + @app.get("/") + def root(): + return { + "message": f"Welcome to {settings.APP_NAME} API", + "docs": "/docs", + "health": "/api/health", + } +app.include_router(profile_router) \ No newline at end of file diff --git a/backend/app/models.py b/backend/app/models.py index b7f714c8..25587fc0 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -17,7 +17,6 @@ Text, Boolean, Enum as SQLAlchemyEnum, - UniqueConstraint, ) from sqlalchemy.types import TypeDecorator, CHAR from sqlalchemy.dialects.postgresql import UUID as PG_UUID @@ -75,7 +74,7 @@ def _get_cipher(self): from app.config import get_settings settings = get_settings() key = base64.urlsafe_b64encode( - hashlib.sha256(settings.FIELD_ENCRYPTION_KEY.encode()).digest() + hashlib.sha256(settings.SECRET_KEY.encode()).digest() ) return Fernet(key) @@ -107,12 +106,6 @@ class UserRole(str, enum.Enum): admin = "admin" -class WorkspaceRole(str, enum.Enum): - admin = "admin" - editor = "editor" - viewer = "viewer" - - class User(Base): """ Represents a registered user within the system. @@ -170,11 +163,6 @@ class User(Base): back_populates="user", cascade="all, delete-orphan", ) - workspace_memberships = relationship( - "WorkspaceMember", - back_populates="user", - cascade="all, delete-orphan", - ) class ApiKey(Base): @@ -200,11 +188,11 @@ class ApiKey(Base): class WorkspaceInvitation(Base): __tablename__ = "workspace_invitations" - id = Column(GUID, primary_key=True, default=uuid.uuid4) + id = Column(String, primary_key=True, default=generate_uuid) email = Column(String(120), nullable=False, index=True) token_hash = Column(String(255), nullable=False, unique=True, index=True) inviter_id = Column( - GUID, + String, ForeignKey("users.id"), nullable=False, index=True, @@ -217,88 +205,6 @@ class WorkspaceInvitation(Base): inviter = relationship("User") -class Workspace(Base): - __tablename__ = "workspaces" - - id = Column(GUID, primary_key=True, default=uuid.uuid4) - - name = Column(String(255), nullable=False) - - created_by = Column( - GUID, - ForeignKey("users.id"), - nullable=False, - index=True, - ) - - created_at = Column( - DateTime, - default=lambda: datetime.now(timezone.utc), - ) - - creator = relationship("User") - - members = relationship( - "WorkspaceMember", - back_populates="workspace", - cascade="all, delete-orphan", - ) - - -class WorkspaceMember(Base): - __tablename__ = "workspace_members" - - __table_args__ = ( - UniqueConstraint( - "workspace_id", - "user_id", - name="uq_workspace_member", - ), - ) - - id = Column( - GUID, - primary_key=True, - default=uuid.uuid4, - ) - - workspace_id = Column( - GUID, - ForeignKey("workspaces.id"), - nullable=False, - index=True, - ) - - user_id = Column( - GUID, - ForeignKey("users.id"), - nullable=False, - index=True, - ) - - role = Column( - SQLAlchemyEnum(WorkspaceRole), - nullable=False, - default=WorkspaceRole.viewer, - server_default="viewer", - ) - - joined_at = Column( - DateTime, - default=lambda: datetime.now(timezone.utc), - ) - - workspace = relationship( - "Workspace", - back_populates="members", - ) - - user = relationship( - "User", - back_populates="workspace_memberships", - ) - - class ChatSession(Base): """ Groups chat messages into logical sessions/threads. @@ -327,7 +233,7 @@ class Document(Base): id = Column(GUID, primary_key=True, default=uuid.uuid4) user_id = Column(GUID, ForeignKey("users.id"), nullable=False, index=True) - filename = Column(String(255), nullable=False, index=True) + filename = Column(String(255), nullable=False) original_name = Column(String(255), nullable=False) file_size = Column(Integer, default=0) page_count = Column(Integer, default=0) @@ -348,14 +254,6 @@ class Document(Base): drive_synced_at = Column(DateTime, nullable=True) is_deleted = Column(Boolean, default=False, nullable=False, index=True) deleted_at = Column(DateTime, nullable=True) - processing_progress = Column(Integer, default=0) - processing_stage = Column(String(20), default="queued") - retry_count = Column(Integer, default=0) - last_error_traceback = Column(Text, nullable=True) - processing_started_at = Column(DateTime, nullable=True) - completed_at = Column(DateTime, nullable=True) - extracted_urls = Column(Text, nullable=True) - keywords = Column(Text, nullable=True) # Relationships owner = relationship("User", back_populates="documents") diff --git a/backend/app/observability.py b/backend/app/observability.py index b9826623..52adb3df 100644 --- a/backend/app/observability.py +++ b/backend/app/observability.py @@ -9,33 +9,7 @@ from fastapi import FastAPI from prometheus_client import Gauge -from prometheus_fastapi_instrumentator import Instrumentator, routing -from starlette.routing import Match - -# ── Workaround for FastAPI 0.135+ and prometheus-fastapi-instrumentator 8.0.0 ── -# Newer FastAPI versions include _IncludedRouter objects in app.routes which -# lack a '.path' attribute, causing AttributeErrors during instrumentation. -def _patched_get_route_name(scope, routes, route_name=None): - """Safe version of _get_route_name that handles routes without a .path attribute.""" - for route in routes: - try: - match, child_scope = route.matches(scope) - except Exception: - continue - - if match == Match.FULL: - # If we have a full match and the route has a path, use it and return early. - # This matches Starlette's behavior where the first matching route wins. - if hasattr(route, "path"): - return route.path - elif match == Match.PARTIAL and hasattr(route, "routes"): - # Recursive call for nested routes (e.g. Mounts) - route_name = _patched_get_route_name(child_scope, route.routes, route_name) - if route_name: - return route_name - return route_name - -routing._get_route_name = _patched_get_route_name +from prometheus_fastapi_instrumentator import Instrumentator APP_PROCESS_RSS_BYTES = Gauge( "app_process_resident_memory_bytes", @@ -70,232 +44,3 @@ def setup_prometheus_metrics(app: FastAPI) -> Instrumentator: ) app.state.prometheus_instrumentator = instrumentator return instrumentator - - -# ── Structured JSON Logging Implementation with Loguru ── - -import os -import time -import uuid -import json -import logging -import contextvars -from loguru import logger -from fastapi import Request -from starlette.middleware.base import BaseHTTPMiddleware -from app.config import get_settings - -# ── Context Variables for request-local structured logs ── -request_id_var = contextvars.ContextVar("request_id", default="") -user_id_var = contextvars.ContextVar("user_id", default="") -upload_filename_var = contextvars.ContextVar("upload_filename", default="") -upload_filesize_var = contextvars.ContextVar("upload_filesize", default=None) -query_text_var = contextvars.ContextVar("query_text", default="") -chunks_retrieved_var = contextvars.ContextVar("chunks_retrieved", default=None) - - -class InterceptHandler(logging.Handler): - """Logs from Python standard logging are redirected to Loguru.""" - - def emit(self, record): - try: - level = logger.level(record.levelname).name - except ValueError: - level = record.levelno - - frame, depth = sys._getframe(6), 6 - while frame and frame.f_code.co_filename == logging.__file__: - frame = frame.f_back - depth += 1 - - logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage()) - - -def json_serializer(record): - """Custom format function to serialize logs as clean JSON.""" - log_record = { - "timestamp": record["time"].isoformat(), - "level": record["level"].name, - "message": record["message"], - "module": record["module"], - "function": record["function"], - "line": record["line"], - } - - # Inject extra attributes (bound context, patch attributes) - if record["extra"]: - for key, val in record["extra"].items(): - if key != "serialized": - log_record[key] = val - - # Include formatted exception if any - if record["exception"]: - exception = record["exception"] - log_record["exception"] = f"{exception.type.__name__}: {exception.value}" - - record["extra"]["serialized"] = json.dumps(log_record) - return "{extra[serialized]}\n" - - -def setup_logging(): - """Setup and configure Loguru logging framework.""" - settings = get_settings() - - # Determine log level - log_level = settings.LOG_LEVEL - if not log_level: - log_level = "DEBUG" if settings.ENVIRONMENT == "development" else "INFO" - - # Remove default handlers - logger.remove() - - # Configure global patcher to inject ContextVars into extra - def patch_record(record): - req_id = request_id_var.get() - if req_id: - record["extra"]["request_id"] = req_id - - u_id = user_id_var.get() - if u_id: - record["extra"]["user_id"] = u_id - - fn = upload_filename_var.get() - if fn: - record["extra"]["filename"] = fn - - fs = upload_filesize_var.get() - if fs is not None: - record["extra"]["file_size"] = fs - - q = query_text_var.get() - if q: - record["extra"]["query"] = q - - chunks = chunks_retrieved_var.get() - if chunks is not None: - record["extra"]["chunks_retrieved"] = chunks - - logger.configure(patcher=patch_record) - - # Add stdout handler - logger.add( - sys.stdout, - format=json_serializer, - level=log_level, - backtrace=True, - diagnose=True, - ) - - # Add file handler with rotation and retention - os.makedirs(os.path.dirname(settings.LOG_FILE), exist_ok=True) - logger.add( - settings.LOG_FILE, - format=json_serializer, - level=log_level, - rotation="10 MB", - retention="10 days", - compression="zip", - backtrace=True, - diagnose=True, - ) - - # Intercept standard library logging - logging.basicConfig(handlers=[InterceptHandler()], level=0, force=True) - - # Redirect Uvicorn logs to loguru - for name in ("uvicorn", "uvicorn.asgi", "uvicorn.access", "uvicorn.error"): - logging_logger = logging.getLogger(name) - logging_logger.handlers = [InterceptHandler()] - logging_logger.propagate = False - - logger.info(f"Logging initialized with level: {log_level}, log file: {settings.LOG_FILE}") - - -class StructuredLoggingMiddleware(BaseHTTPMiddleware): - """Middleware to inject context variables and log HTTP requests in structured JSON format.""" - - async def dispatch(self, request: Request, call_next): - start_time = time.perf_counter() - response = None - - # Unique Request ID per request - request_id = request.headers.get("X-Request-ID") or uuid.uuid4().hex - - # Set request-local context variables - token_req = request_id_var.set(request_id) - token_user = user_id_var.set("") - token_filename = upload_filename_var.set("") - token_filesize = upload_filesize_var.set(None) - token_query = query_text_var.set("") - token_chunks = chunks_retrieved_var.set(None) - - # Also store on request state for reliable retrieval - request.state.request_id = request_id - request.state.user_id = "" - request.state.filename = "" - request.state.filesize = None - request.state.query = "" - request.state.chunks_retrieved = None - - try: - response = await call_next(request) - except Exception as exc: - duration = time.perf_counter() - start_time - # Log exception with request details - logger.opt(exception=exc).error( - f"HTTP request failed: {request.method} {request.url.path} - Exception: {str(exc)}", - extra={ - "method": request.method, - "path": request.url.path, - "status_code": 500, - "response_time_ms": round(duration * 1000, 2), - } - ) - raise exc from None - finally: - duration = time.perf_counter() - start_time - - # Read state variables and set contextvars again in case they didn't propagate back - u_id = getattr(request.state, "user_id", "") - fn = getattr(request.state, "filename", "") - fs = getattr(request.state, "filesize", None) - q = getattr(request.state, "query", "") - chunks = getattr(request.state, "chunks_retrieved", None) - - if u_id: user_id_var.set(u_id) - if fn: upload_filename_var.set(fn) - if fs is not None: upload_filesize_var.set(fs) - if q: query_text_var.set(q) - if chunks is not None: chunks_retrieved_var.set(chunks) - - # Log final response details - # We don't log health check endpoints or metrics endpoints in standard INFO logs to reduce clutter, - # but we can log them at DEBUG level. - is_health_or_metrics = request.url.path in ("/api/health", "/health", "/metrics") - log_fn = logger.debug if is_health_or_metrics else logger.info - - status_code = getattr(response, "status_code", 500) - log_fn( - f"HTTP request completed: {request.method} {request.url.path} - {status_code}", - extra={ - "method": request.method, - "path": request.url.path, - "status_code": status_code, - "response_time_ms": round(duration * 1000, 2), - } - ) - - # Add X-Request-ID to response headers - if response is not None: - response.headers["X-Request-ID"] = request_id - - # Reset ContextVars to avoid leaking - request_id_var.reset(token_req) - user_id_var.reset(token_user) - upload_filename_var.reset(token_filename) - upload_filesize_var.reset(token_filesize) - query_text_var.reset(token_query) - chunks_retrieved_var.reset(token_chunks) - - return response - diff --git a/backend/app/rag/agent.py b/backend/app/rag/agent.py index b1f0f047..ceae0a7e 100644 --- a/backend/app/rag/agent.py +++ b/backend/app/rag/agent.py @@ -6,34 +6,25 @@ import json from typing import List, Dict, Any, Optional, Generator +from sympy import python + from huggingface_hub import InferenceClient from langchain_classic.agents import create_react_agent, AgentExecutor from langchain_core.prompts import PromptTemplate -from langchain_huggingface import HuggingFaceEndpoint -from langchain_huggingface.chat_models import ChatHuggingFace +from langchain_huggingface import HuggingFaceEndpoint, ChatHuggingFace from app.config import get_settings from app.rag.retriever import retrieve from app.rag.graph_retriever import get_entity_context -from app.rag.prompts import AGENT_SYSTEM_PROMPT, MULTI_DOC_COMPARISON_GUIDANCE -from app.exceptions import ExternalServiceException +from app.rag.prompts import AGENT_SYSTEM_PROMPT from app.rag.security import MALFORMED_OUTPUT_MESSAGE, OutputParserError, parse_agent_output from app.rag.tools import PDFSearchTool, MathTool, WebSearchTool from app.rag.tracing import trace_function -from app.rag.keywords import extract_keywords logger = logging.getLogger(__name__) settings = get_settings() -def persist_document_keywords(document, chunks, db) -> None: - """Extract TF-IDF keywords after indexing and persist on the document row.""" - raw_texts = [c["text"] for c in chunks] - kws = extract_keywords(raw_texts, top_n=10) - document.keywords = json.dumps(kws) - db.add(document) - - def get_llm_client(hf_token: Optional[str] = None) -> InferenceClient: """Create a HuggingFace InferenceClient per-request.""" @@ -61,7 +52,6 @@ def _format_chat_history(messages: List[Dict[str, str]]) -> str: def get_agent_executor( user_id: str, document_id: Optional[str] = None, - document_ids: Optional[List[str]] = None, hf_token: Optional[str] = None, top_k: Optional[int] = None, chat_history: Optional[List[Dict[str, str]]] = None, @@ -69,7 +59,7 @@ def get_agent_executor( """Initialize the LangChain ReAct agent executor.""" # Initialize tools - pdf_tool = PDFSearchTool(user_id=user_id, document_id=document_id, document_ids=document_ids, top_k=top_k) + pdf_tool = PDFSearchTool(user_id=user_id, document_id=document_id, top_k=top_k) tools = [pdf_tool, MathTool(), WebSearchTool()] # Initialize LLM @@ -91,14 +81,7 @@ def get_agent_executor( chat_llm = ChatHuggingFace(llm=llm) # Setup Agent - agent_prompt_text = AGENT_SYSTEM_PROMPT - if document_ids and len(document_ids) > 1: - agent_prompt_text = agent_prompt_text.replace( - "Begin!", - MULTI_DOC_COMPARISON_GUIDANCE + "\nBegin!", - 1, - ) - prompt = PromptTemplate.from_template(agent_prompt_text) + prompt = PromptTemplate.from_template(AGENT_SYSTEM_PROMPT) agent = create_react_agent(chat_llm, tools, prompt) executor = AgentExecutor( @@ -135,7 +118,6 @@ def generate_answer( question: str, user_id: str, document_id: Optional[str] = None, - document_ids: Optional[List[str]] = None, hf_token: Optional[str] = None, top_k: Optional[int] = None, chat_history: Optional[List[Dict[str, str]]] = None, @@ -163,7 +145,7 @@ def generate_answer( # ── Run Agent ──────────────────────────────────── try: - executor, pdf_tool, formatted_history = get_agent_executor(user_id, document_id, document_ids, hf_token, top_k, chat_history) + executor, pdf_tool, formatted_history = get_agent_executor(user_id, document_id, hf_token, top_k, chat_history) result = executor.invoke({"input": question, "chat_history": formatted_history}) raw_answer = result.get("output", "") @@ -188,12 +170,12 @@ def generate_answer( return {"answer": answer, "sources": sources} - except (OutputParserError, ValueError) as e: - logger.warning(f"Agent output error: {e}") - return {"answer": MALFORMED_OUTPUT_MESSAGE, "sources": []} except Exception as e: logger.error(f"Agent execution error: {e}") - raise ExternalServiceException("HuggingFace", str(e)) from e + return { + "answer": f"I encountered an error while processing your request: {str(e)}", + "sources": [] + } @trace_function( @@ -208,7 +190,6 @@ def generate_answer_stream( question: str, user_id: str, document_id: Optional[str] = None, - document_ids: Optional[List[str]] = None, hf_token: Optional[str] = None, top_k: Optional[int] = None, chat_history: Optional[List[Dict[str, str]]] = None, @@ -237,7 +218,7 @@ def generate_answer_stream( # ── Run Agent ──────────────────────────────────── try: - executor, pdf_tool, formatted_history = get_agent_executor(user_id, document_id, document_ids, hf_token, top_k, chat_history) + executor, pdf_tool, formatted_history = get_agent_executor(user_id, document_id, hf_token, top_k, chat_history) sources_sent = False diff --git a/backend/app/rag/bm25.py b/backend/app/rag/bm25.py index a5c1150f..82298bf5 100644 --- a/backend/app/rag/bm25.py +++ b/backend/app/rag/bm25.py @@ -6,7 +6,6 @@ import glob import pickle import logging -import re from typing import List, Dict, Any, Optional from app.config import get_settings @@ -31,8 +30,9 @@ def get_bm25_path(user_id: str, document_id: str) -> str: return os.path.join(get_bm25_dir(user_id), f"{document_id}.pkl") def tokenize(text: str) -> List[str]: - """Tokenize by converting to lowercase and extracting all alphanumeric words.""" - return re.findall(r'\w+', text.lower()) + """Simple tokenization for BM25.""" + # Convert to lowercase and split by whitespace + return text.lower().split() def store_bm25_index(chunks: List[Dict[str, Any]], document_id: str, filename: str, user_id: str): """ @@ -52,10 +52,7 @@ def store_bm25_index(chunks: List[Dict[str, Any]], document_id: str, filename: s # Format chunks to match vectorstore output formatted_chunks = [] for chunk in chunks: - chunk_idx = chunk.get("chunk_index") - chunk_id = f"{document_id}_{chunk_idx}" if chunk_idx is not None else None formatted_chunks.append({ - "id": chunk_id, "text": chunk["text"], "filename": filename, "document_id": document_id, @@ -111,7 +108,6 @@ def query_bm25( query: str, user_id: str, document_id: Optional[str] = None, - document_ids: Optional[List[str]] = None, top_k: int = 10, ) -> List[Dict[str, Any]]: """ @@ -126,17 +122,7 @@ def query_bm25( path = get_bm25_path(user_id, document_id) return _query_single_index(path, tokenized_query, top_k) - if document_ids: - all_results = [] - for doc_id in document_ids: - path = get_bm25_path(user_id, doc_id) - if os.path.exists(path): - results = _query_single_index(path, tokenized_query, top_k) - all_results.extend(results) - all_results.sort(key=lambda x: x["score"], reverse=True) - return all_results[:top_k] - - # If no document_id and no document_ids, query all documents for this user + # If no document_id, query all documents for this user user_dir = get_bm25_dir(user_id) all_results = [] diff --git a/backend/app/rag/chunker.py b/backend/app/rag/chunker.py index 96d337d1..74579da3 100644 --- a/backend/app/rag/chunker.py +++ b/backend/app/rag/chunker.py @@ -6,13 +6,11 @@ import re import fitz # PyMuPDF import docx -import logging from typing import List, Dict, Any from langchain_text_splitters import RecursiveCharacterTextSplitter from app.config import get_settings settings = get_settings() -logger = logging.getLogger(__name__) def _is_word_inside_bbox(word: Dict[str, Any], bbox: tuple) -> bool: @@ -24,12 +22,11 @@ def _is_word_inside_bbox(word: Dict[str, Any], bbox: tuple) -> bool: def _words_to_text(words: List[Dict[str, Any]], line_tolerance: float = 3.0) -> str: - """Rebuild readable text from positioned pdfplumber words, preserving multi-column reading order.""" + """Rebuild readable text from positioned pdfplumber words.""" if not words: return "" - # 1. Group words into horizontal lines based on vertical proximity - sorted_words = sorted(words, key=lambda item: (float(item["top"]), item["x0"])) + sorted_words = sorted(words, key=lambda item: (round(float(item["top"]) / line_tolerance), item["x0"])) lines: List[List[Dict[str, Any]]] = [] for word in sorted_words: @@ -43,120 +40,11 @@ def _words_to_text(words: List[Dict[str, Any]], line_tolerance: float = 3.0) -> else: lines.append([word]) - # 2. Split each line into segments based on horizontal gaps (column gutters) - GAP_THRESHOLD = 20.0 - line_segments_list = [] - for line in lines: - sorted_line_words = sorted(line, key=lambda item: item["x0"]) - segments_in_line = [] - current_seg = [sorted_line_words[0]] - - for word in sorted_line_words[1:]: - prev_word = current_seg[-1] - gap = float(word["x0"]) - float(prev_word["x1"]) - if gap > GAP_THRESHOLD: - segments_in_line.append(current_seg) - current_seg = [word] - else: - current_seg.append(word) - segments_in_line.append(current_seg) - line_segments_list.append(segments_in_line) - - # 3. Detect global vertical gutters from lines with multiple layout segments - gutter_intervals = [] - for seg_list in line_segments_list: - if len(seg_list) > 1: - for i in range(len(seg_list) - 1): - x1_prev = max(float(w["x1"]) for w in seg_list[i]) - x0_next = min(float(w["x0"]) for w in seg_list[i+1]) - if x0_next > x1_prev: - gutter_intervals.append((x1_prev, x0_next)) - - significant_gutter_centers = [] - if gutter_intervals: - sorted_gutters = sorted(gutter_intervals, key=lambda g: g[0]) - current_gutter_groups = [] - for g in sorted_gutters: - inserted = False - for group in current_gutter_groups: - group_x0 = max(item[0] for item in group) - group_x1 = min(item[1] for item in group) - if max(g[0], group_x0) < min(g[1], group_x1): - group.append(g) - inserted = True - break - if not inserted: - current_gutter_groups.append([g]) - - num_multi_seg_lines = sum(1 for seg_list in line_segments_list if len(seg_list) > 1) - min_group_size = max(2, int(num_multi_seg_lines * 0.15)) - - for group in current_gutter_groups: - if len(group) >= min_group_size: - centers = [(g[0] + g[1]) / 2.0 for g in group] - significant_gutter_centers.append(sum(centers) / len(centers)) - significant_gutter_centers.sort() - - # 4. Flatten segments and capture absolute structural bounds - segment_objs = [] - for seg_list in line_segments_list: - for seg in seg_list: - seg_x0 = min(float(w["x0"]) for w in seg) - seg_x1 = max(float(w["x1"]) for w in seg) - seg_top = min(float(w["top"]) for w in seg) - seg_bottom = max(float(w["bottom"]) for w in seg) - seg_text = " ".join(w["text"] for w in seg) - segment_objs.append({ - "x0": seg_x0, - "x1": seg_x1, - "top": seg_top, - "bottom": seg_bottom, - "text": seg_text - }) - - # 5. Sort segments vertically into horizontal layout bands divided by full-width text structures - segment_objs.sort(key=lambda s: s["top"]) - - final_segments = [] - current_band = [] - GUTTER_TOLERANCE = 3.0 - - for seg in segment_objs: - crosses_gutter = False - for center in significant_gutter_centers: - if (seg["x0"] + GUTTER_TOLERANCE) < center < (seg["x1"] - GUTTER_TOLERANCE): - crosses_gutter = True - break - - if crosses_gutter: - if current_band: - def get_col_idx(s): - idx = 0 - s_mid = (s["x0"] + s["x1"]) / 2.0 - for c in significant_gutter_centers: - if s_mid > c: - idx += 1 - return idx - current_band.sort(key=lambda s: (get_col_idx(s), s["top"])) - final_segments.extend(current_band) - current_band = [] - final_segments.append(seg) - else: - current_band.append(seg) - - if current_band: - def get_col_idx(s): - idx = 0 - s_mid = (s["x0"] + s["x1"]) / 2.0 - for c in significant_gutter_centers: - if s_mid > c: - idx += 1 - return idx - current_band.sort(key=lambda s: (get_col_idx(s), s["top"])) - final_segments.extend(current_band) - - text_lines = [seg["text"] for seg in final_segments if seg["text"].strip()] - return "\n".join(text_lines) + text_lines = [ + " ".join(item["text"] for item in sorted(line, key=lambda item: item["x0"])) + for line in lines + ] + return "\n".join(line for line in text_lines if line.strip()) def _clean_table_cell(cell: Any) -> str: @@ -189,167 +77,28 @@ def fmt(row: List[str]) -> str: def extract_pdf(filepath: str) -> List[Dict[str, Any]]: - """Extract PDF text while preserving tables as separate chunks. - - Extraction is attempted in order of richness: - 1. Unstructured (tables + layout) - 2. pdfplumber (tables + multi-column) - 3. PyMuPDF native text - 4. OCR fallback for image-only pages (pytesseract or easyocr) - - Each stage only runs if the previous one raises an exception OR - produces no text at all (e.g. a purely scanned PDF). - """ - pages: List[Dict[str, Any]] = [] - + """Extract PDF text while preserving tables as separate bbox-aware chunks.""" try: - pages = extract_pdf_with_unstructured(filepath) - except Exception as e: - logger.warning(f"Unstructured extraction failed, falling back: {e}") - - if not pages: - try: - pages = extract_pdf_with_tables(filepath) - except Exception as e2: - logger.warning(f"pdfplumber extraction failed, falling back: {e2}") - - if not pages: - try: - pages = extract_pdf_with_pymupdf(filepath) - except Exception as e3: - logger.warning(f"PyMuPDF extraction failed, falling back to OCR: {e3}") - - # If still no text, run a full OCR pass for scanned/image-only PDFs. - if not pages: - logger.info( - "All text extractors returned empty for '%s' — running full OCR pass", - filepath, - ) - try: - from app.rag.ocr import extract_pdf_with_ocr - pages = extract_pdf_with_ocr(filepath) - except Exception as exc: - logger.warning("OCR pass failed for '%s': %s", filepath, exc) - - return pages + return extract_pdf_with_tables(filepath) + except ImportError: + return extract_pdf_with_pymupdf(filepath) def extract_pdf_with_pymupdf(filepath: str) -> List[Dict[str, Any]]: - """Fallback PDF extraction with page numbers using PyMuPDF. - - For pages with no selectable text (scanned/image-only pages) OCR is - attempted automatically using the configured OCR backend. - """ - from app.rag.ocr import MIN_TEXT_CHARS, OCR_BACKEND, ocr_page - + """Fallback PDF extraction with page numbers using PyMuPDF.""" doc = fitz.open(filepath) pages = [] - try: - for page_num, page in enumerate(doc): - text = page.get_text().strip() - - if len(text) >= MIN_TEXT_CHARS: - pages.append({ - "text": text, - "page": page_num + 1, - "chunk_type": "text", - "ocr": False, - }) - continue - - # No selectable text — attempt OCR - logger.info( - "Page %d of '%s' is image-only — running OCR (backend=%s)", - page_num + 1, - filepath, - OCR_BACKEND, - ) - try: - ocr_text = ocr_page(page) - if ocr_text: - pages.append({ - "text": ocr_text, - "page": page_num + 1, - "chunk_type": "text", - "ocr": True, - }) - else: - logger.warning( - "OCR returned no text for page %d of '%s'", - page_num + 1, - filepath, - ) - except ImportError: - logger.warning( - "OCR backend '%s' not installed — skipping image-only page %d", - OCR_BACKEND, - page_num + 1, - ) - except Exception as exc: - logger.warning( - "OCR failed for page %d of '%s': %s", - page_num + 1, - filepath, - exc, - ) - finally: - doc.close() - - return pages - - -def extract_pdf_with_unstructured(filepath: str) -> List[Dict[str, Any]]: - """Use Unstructured to partition PDF into elements and extract tables.""" - try: - from unstructured.partition.pdf import partition_pdf - from unstructured.documents.elements import Table - except Exception as e: - raise ImportError("unstructured not available") from e - - elements = partition_pdf(filename=filepath) - pages: List[Dict[str, Any]] = [] - table_idx = 0 - - for elem in elements: - elem_type = getattr(elem, "element_type", None) or elem.__class__.__name__ - page_num = None - if hasattr(elem, "page_number"): - page_num = getattr(elem, "page_number") - elif getattr(elem, "metadata", None): - page_num = elem.metadata.get("page_number") or elem.metadata.get("page") - page_num = int(page_num) if page_num else 1 - - if isinstance(elem, Table) or (isinstance(elem_type, str) and elem_type.lower() == "table"): - rows = [] - for raw_row in getattr(elem, "rows", []) or []: - row = [] - for cell in raw_row: - if isinstance(cell, (list, tuple)): - cell_text = " ".join(getattr(c, "text", str(c)) for c in cell) - else: - cell_text = getattr(cell, "text", str(cell)) - row.append(cell_text) - rows.append(row) - - table_text = _table_to_markdown(rows) - if table_text.strip(): - pages.append({ - "text": table_text, - "page": page_num, - "chunk_type": "table", - "table_index": table_idx, - }) - table_idx += 1 - else: - text = getattr(elem, "text", str(elem) if elem else "") - if text and text.strip(): - pages.append({ - "text": text, - "page": page_num, - "chunk_type": "text", - }) + for page_num, page in enumerate(doc): + text = page.get_text() + if text.strip(): + pages.append({ + "text": text, + "page": page_num + 1, + "chunk_type": "text", + }) + doc.close() return pages @@ -381,6 +130,7 @@ def extract_pdf_with_tables(filepath: str) -> List[Dict[str, Any]]: for table_index, table in enumerate(tables): table_text = _table_to_markdown(table.extract() or []) if table_text.strip(): + # Normalize table bbox: [x0/W, y0/H, x1/W, y1/H] W, H = float(page.width), float(page.height) normalized_bbox = [ round(float(table.bbox[0]) / W, 4), @@ -399,61 +149,32 @@ def extract_pdf_with_tables(filepath: str) -> List[Dict[str, Any]]: return pages -def extract_pdf_images( - doc_or_path: Any, - min_width: int = 50, - min_height: int = 50, - min_size: int = 10240, -) -> Any: - """Generator to yield extracted images from a PDF page-by-page.""" - if not doc_or_path: - return +def extract_pdf_images(filepath: str) -> List[Dict[str, Any]]: + """Extract images from a PDF and return list of dicts with image bytes and page number. - is_path = isinstance(doc_or_path, str) - doc = None - if is_path: - try: - doc = fitz.open(doc_or_path) - except Exception as e: - logger.warning(f"Could not open PDF with fitz for image extraction: {e}") - return - else: - doc = doc_or_path + Each entry: {"image_bytes": b"...", "page": int} + """ + images = [] + doc = fitz.open(filepath) - try: - for page_num, page in enumerate(doc): - processed_xrefs = set() - for img in page.get_images(full=True): - xref = img[0] - if xref in processed_xrefs: - continue - processed_xrefs.add(xref) - - try: - pix = fitz.Pixmap(doc, xref) - width, height = pix.width, pix.height - - if pix.n >= 4: - pix = fitz.Pixmap(fitz.csRGB, pix) - - img_bytes = pix.tobytes("png") - - if width < min_width or height < min_height or len(img_bytes) < min_size: - del img_bytes - del pix - continue - - yield { - "image_bytes": img_bytes, - "page": page_num + 1, - "width": width, - "height": height, - } - except Exception: - continue - finally: - if is_path and doc: - doc.close() + for page_num, page in enumerate(doc): + # get_images returns a list of tuples where first item is xref + for img in page.get_images(full=True): + xref = img[0] + try: + pix = fitz.Pixmap(doc, xref) + # Convert to RGB if it's CMYK or has alpha + if pix.n >= 4: + pix = fitz.Pixmap(fitz.csRGB, pix) + + img_bytes = pix.tobytes("png") + images.append({"image_bytes": img_bytes, "page": page_num + 1}) + except Exception: + # ignore extracting this image + continue + + doc.close() + return images def extract_docx(filepath: str) -> List[Dict[str, Any]]: @@ -482,6 +203,7 @@ def extract_txt(filepath: str) -> List[Dict[str, Any]]: if is_table_line: if not in_table: + # flush any accumulated text first if current_text_lines: chunk_text = "\n".join(current_text_lines).strip() if chunk_text: @@ -491,6 +213,7 @@ def extract_txt(filepath: str) -> List[Dict[str, Any]]: table_lines.append(line) else: if in_table: + # flush the table table_text = "\n".join(table_lines).strip() if table_text: chunks.append({"text": table_text, "page": 1, "chunk_type": "table"}) @@ -498,6 +221,7 @@ def extract_txt(filepath: str) -> List[Dict[str, Any]]: in_table = False current_text_lines.append(line) + # flush whatever's left if in_table and table_lines: chunks.append({"text": "\n".join(table_lines).strip(), "page": 1, "chunk_type": "table"}) elif current_text_lines: @@ -507,13 +231,22 @@ def extract_txt(filepath: str) -> List[Dict[str, Any]]: return chunks - +# Change the chunk_document function input to take a file path and optional chunk size and overlap parameters. def chunk_document(filepath: str, chunk_size: int = None, chunk_overlap: int = None) -> List[Dict[str, Any]]: - """Load a document, extract text per page, and split into semantic chunks.""" + """ + Load a document, extract text per page, and split into semantic chunks. + Accepts a file path and optional chunk size and overlap parameters. + If chunk size and overlap are not provided, defaults from settings will be used. + Returns list of dicts with 'text', 'page', and 'chunk_index'. + """ ext = filepath.rsplit(".", 1)[-1].lower() + images = [] + # ── Extract text by file type ──────────────────── if ext == "pdf": pages = extract_pdf(filepath) + # also extract images for later captioning/embedding + images = extract_pdf_images(filepath) elif ext == "docx": pages = extract_docx(filepath) elif ext in ("txt", "md"): @@ -521,152 +254,101 @@ def chunk_document(filepath: str, chunk_size: int = None, chunk_overlap: int = N else: raise ValueError(f"Unsupported file type: {ext}") - pdf_doc = None - if ext == "pdf": - try: - pdf_doc = fitz.open(filepath) - except Exception as e: - logger.warning(f"Could not open PDF with fitz: {e}") - if not pages: - if not pdf_doc or len(pdf_doc) == 0: - if pdf_doc: - pdf_doc.close() - return [] + return [] + # Set chunk size and chunk overlap with defaults if not provided if not chunk_size: chunk_size = settings.CHUNK_SIZE if not chunk_overlap: chunk_overlap = settings.CHUNK_OVERLAP + # ── LangChain recursive splitter ───────────────── splitter = RecursiveCharacterTextSplitter( - chunk_size=chunk_size, - chunk_overlap=chunk_overlap, + chunk_size=chunk_size, # Allow custom chunk size to be passed in for embedding + chunk_overlap=chunk_overlap, # Allow custom chunk overlap to be passed in for embedding separators=["\n\n", "\n", ". ", " ", ""], length_function=len, ) all_chunks = [] chunk_index = 0 + pdf_doc = None + + if ext == "pdf": + try: + pdf_doc = fitz.open(filepath) + except Exception as e: + import logging + logger = logging.getLogger(__name__) + logger.warning(f"Could not open PDF with fitz for bbox extraction: {e}") try: - max_page_in_data = max(page_data["page"] for page_data in pages) if pages else 0 - total_pages = max(max_page_in_data, len(pdf_doc) if pdf_doc else 0) + for page_data in pages: + text = page_data["text"] + page_num = page_data["page"] + chunk_type = page_data.get("chunk_type", "text") + + if chunk_type == "table": + all_chunks.append({ + "text": text.strip(), + "page": page_num, + "chunk_index": chunk_index, + "chunk_type": "table", + "bbox": page_data.get("bbox", ""), + "table_index": page_data.get("table_index", 0), + }) + chunk_index += 1 + continue - image_iter = None - next_image = None - if pdf_doc: - try: - image_iter = iter(extract_pdf_images(pdf_doc)) - next_image = next(image_iter) - except StopIteration: - next_image = None - except Exception as e: - logger.warning(f"Could not initialize image iterator: {e}") - - pages_by_num = {} - for p_data in pages: - pages_by_num.setdefault(p_data["page"], []).append(p_data) - - for page_num in range(1, total_pages + 1): - page_data_list = pages_by_num.get(page_num, []) - for page_data in page_data_list: - text = page_data["text"] - chunk_type = page_data.get("chunk_type", "text") - - if chunk_type == "table": - all_chunks.append({ - "text": text.strip(), + # Split this page's text + splits = splitter.split_text(text) + + for split_text in splits: + if split_text.strip(): + chunk = { + "text": split_text.strip(), "page": page_num, "chunk_index": chunk_index, - "chunk_type": "table", - "bbox": page_data.get("bbox", ""), - "table_index": page_data.get("table_index", 0), - }) - chunk_index += 1 - continue - - splits = splitter.split_text(text) - - for split_text in splits: - if split_text.strip(): - chunk = { - "text": split_text.strip(), - "page": page_num, - "chunk_index": chunk_index, - "chunk_type": chunk_type, - } - - if pdf_doc and page_num <= len(pdf_doc): - try: - page_obj = pdf_doc[page_num - 1] - rects = page_obj.search_for(split_text.strip()) - if rects: - W, H = float(page_obj.rect.width), float(page_obj.rect.height) - norm_rects = [ - [ - round(r.x0 / W, 4), - round(r.y0 / H, 4), - round(r.x1 / W, 4), - round(r.y1 / H, 4) - ] - for r in rects + "chunk_type": chunk_type, + } + + # Extract bbox for PDF text chunks + if pdf_doc and page_num <= len(pdf_doc): + try: + page_obj = pdf_doc[page_num - 1] + # Use search_for to find the text on the page + rects = page_obj.search_for(split_text.strip()) + if rects: + W, H = float(page_obj.rect.width), float(page_obj.rect.height) + # Rects can span multiple lines, we store them as a list of normalized bboxes + norm_rects = [ + [ + round(r.x0 / W, 4), + round(r.y0 / H, 4), + round(r.x1 / W, 4), + round(r.y1 / H, 4) ] - chunk["bbox"] = json.dumps(norm_rects) - except Exception as e: - logger.warning(f"Bbox extraction error on page {page_num}: {e}") - - all_chunks.append(chunk) - chunk_index += 1 - - while next_image and next_image["page"] == page_num: - img_bytes = next_image["image_bytes"] - try: - from app.rag.vision import caption_image - caption = caption_image(img_bytes, page=page_num) - - if caption: - all_chunks.append({ - "text": caption, - "page": page_num, - "chunk_index": chunk_index, - "chunk_type": "text", - "is_image": True, - "image_caption": caption, - }) - chunk_index += 1 - except Exception as e: - logger.warning(f"Failed to generate caption for image on page {page_num}: {e}") - fallback_text = f"Image on page {page_num}." - all_chunks.append({ - "text": fallback_text, - "page": page_num, - "chunk_index": chunk_index, - "chunk_type": "text", - "is_image": True, - "image_caption": fallback_text, - }) + for r in rects + ] + chunk["bbox"] = json.dumps(norm_rects) + except Exception as e: + import logging + logger = logging.getLogger(__name__) + logger.warning(f"Bbox extraction error on page {page_num}: {e}") + + all_chunks.append(chunk) chunk_index += 1 - finally: - if next_image: - next_image["image_bytes"] = None - if "img_bytes" in locals(): - del img_bytes - - try: - if image_iter is not None: - next_image = next(image_iter) - else: - next_image = None - except StopIteration: - next_image = None - except Exception as e: - logger.warning(f"Error getting next image from iterator: {e}") - next_image = None - - import gc - gc.collect() + # Attach any images that belong to this page after text chunks for the page + for img in [i for i in images if i["page"] == page_num]: + all_chunks.append({ + "text": "", + "page": page_num, + "chunk_index": chunk_index, + "image_bytes": img["image_bytes"], + }) + chunk_index += 1 finally: if pdf_doc: pdf_doc.close() @@ -684,4 +366,4 @@ def get_page_count(filepath: str) -> int: doc.close() return count - return 1 + return 1 # DOCX, TXT, MD are treated as single-page diff --git a/backend/app/rag/embeddings.py b/backend/app/rag/embeddings.py index b83fcdf4..219741c9 100644 --- a/backend/app/rag/embeddings.py +++ b/backend/app/rag/embeddings.py @@ -1,102 +1,19 @@ """ HuggingFace local embeddings using sentence-transformers. Loads the model once via singleton pattern for efficiency. -Embedding vectors are cached by SHA-256 hash of the input text to avoid -recomputing identical chunks across documents. """ -import hashlib -import json import logging -import os -from typing import List, Optional - +from typing import List from langchain_huggingface import HuggingFaceEmbeddings - from app.config import get_settings from app.rag.tracing import trace_call logger = logging.getLogger(__name__) settings = get_settings() -# ── Singleton embedding model ───────────────────────────────────────────────── +# ── Singleton embedding model ──────────────────────── _embedding_model = None -# ── Embedding cache configuration ──────────────────────────────────────────── -_EMBEDDING_CACHE_TTL: int = int(os.getenv("EMBEDDING_CACHE_TTL", "86400")) # 24 h -_EMBEDDING_LRU_MAX_SIZE: int = int(os.getenv("EMBEDDING_LRU_MAX_SIZE", "512")) - -# ── LRU in-memory fallback (mirrors cache.py pattern) ──────────────────────── -_emb_lru_store: dict = {} -_emb_lru_order: list = [] - - -def _lru_get(key: str) -> Optional[List[float]]: - raw = _emb_lru_store.get(key) - return json.loads(raw) if raw is not None else None - - -def _lru_set(key: str, vector: List[float]) -> None: - if key in _emb_lru_store: - _emb_lru_order.remove(key) - elif len(_emb_lru_store) >= _EMBEDDING_LRU_MAX_SIZE: - oldest = _emb_lru_order.pop(0) - del _emb_lru_store[oldest] - _emb_lru_store[key] = json.dumps(vector) - _emb_lru_order.append(key) - - -# ── Redis helper (reuses the same client from cache.py) ────────────────────── - -def _get_redis(): - """Reuse the Redis client already initialised by cache.py.""" - try: - from app.cache import _get_redis as _cache_get_redis - return _cache_get_redis() - except Exception: - return None - - -# ── Cache key ───────────────────────────────────────────────────────────────── - -def _make_embedding_key(text: str) -> str: - """SHA-256 hash of (model_name, text) so keys are model-scoped.""" - raw = f"{settings.EMBEDDING_MODEL}:{text.strip()}" - return "emb:" + hashlib.sha256(raw.encode("utf-8")).hexdigest() - - -# ── Cache read / write ──────────────────────────────────────────────────────── - -def _cache_get(key: str) -> Optional[List[float]]: - r = _get_redis() - if r is not None: - try: - value = r.get(key) - if value: - logger.debug("Embedding cache HIT (Redis) %s", key[:16]) - return json.loads(value) - except Exception as exc: - logger.warning("Redis embedding GET failed: %s", exc) - - vector = _lru_get(key) - if vector is not None: - logger.debug("Embedding cache HIT (LRU) %s", key[:16]) - return vector - - -def _cache_set(key: str, vector: List[float]) -> None: - r = _get_redis() - if r is not None: - try: - r.setex(key, _EMBEDDING_CACHE_TTL, json.dumps(vector)) - logger.debug("Embedding cache SET (Redis) %s TTL %ds", key[:16], _EMBEDDING_CACHE_TTL) - return - except Exception as exc: - logger.warning("Redis embedding SET failed: %s", exc) - _lru_set(key, vector) - logger.debug("Embedding cache SET (LRU) %s", key[:16]) - - -# ── Singleton embedding model ───────────────────────────────────────────────── def get_embedding_model() -> HuggingFaceEmbeddings: """ @@ -106,7 +23,7 @@ def get_embedding_model() -> HuggingFaceEmbeddings: global _embedding_model if _embedding_model is None: - logger.info("Loading embedding model: %s", settings.EMBEDDING_MODEL) + logger.info(f"Loading embedding model: {settings.EMBEDDING_MODEL}") _embedding_model = HuggingFaceEmbeddings( model_name=settings.EMBEDDING_MODEL, model_kwargs={"device": "cpu"}, @@ -117,68 +34,24 @@ def get_embedding_model() -> HuggingFaceEmbeddings: return _embedding_model -# ── Public API ──────────────────────────────────────────────────────────────── - def embed_texts(texts: List[str]) -> List[List[float]]: - """Embed a batch of texts, serving cached vectors where available. - - For each text: - 1. Check cache (Redis → LRU). - 2. Collect misses into a batch and embed them in one model call. - 3. Write new vectors back to cache and reassemble in original order. - """ - if not texts: - return [] - - keys = [_make_embedding_key(t) for t in texts] - results: List[Optional[List[float]]] = [None] * len(texts) - miss_indices: List[int] = [] - miss_texts: List[str] = [] - - # ── Cache lookup pass ───────────────────────────── - for i, key in enumerate(keys): - cached = _cache_get(key) - if cached is not None: - results[i] = cached - else: - miss_indices.append(i) - miss_texts.append(texts[i]) - - cache_hits = len(texts) - len(miss_indices) - logger.debug( - "embed_texts: %d/%d served from cache, %d to embed", - cache_hits, len(texts), len(miss_indices), + """Embed a batch of texts into vectors.""" + model = get_embedding_model() + return trace_call( + "embed_texts", + lambda: model.embed_documents(texts), + run_type="embedding", + metadata={ + "embedding_model": settings.EMBEDDING_MODEL, + "text_count": len(texts), + }, ) - # ── Embed misses in one batched call ────────────── - if miss_texts: - model = get_embedding_model() - new_vectors: List[List[float]] = trace_call( - "embed_texts", - lambda: model.embed_documents(miss_texts), - run_type="embedding", - metadata={ - "embedding_model": settings.EMBEDDING_MODEL, - "text_count": len(miss_texts), - "cache_hits": cache_hits, - }, - ) - for idx, vector in zip(miss_indices, new_vectors): - _cache_set(keys[idx], vector) - results[idx] = vector - - return results # type: ignore[return-value] - def embed_query(query: str) -> List[float]: - """Embed a single query string, using cache when available.""" - key = _make_embedding_key(query) - cached = _cache_get(key) - if cached is not None: - return cached - + """Embed a single query string.""" model = get_embedding_model() - vector: List[float] = trace_call( + return trace_call( "embed_query", lambda: model.embed_query(query), run_type="embedding", @@ -187,5 +60,3 @@ def embed_query(query: str) -> List[float]: "query_length": len(query), }, ) - _cache_set(key, vector) - return vector diff --git a/backend/app/rag/keywords.py b/backend/app/rag/keywords.py deleted file mode 100644 index 7f1fc2a2..00000000 --- a/backend/app/rag/keywords.py +++ /dev/null @@ -1,54 +0,0 @@ -# backend/app/rag/keywords.py -import math -import re -from collections import Counter -from typing import List - -_STOPWORDS = { - "the","a","an","and","or","but","in","on","at","to","for","of","with", - "is","are","was","were","be","been","being","have","has","had","do","does", - "did","will","would","could","should","may","might","shall","can","need", - "that","this","these","those","it","its","from","by","as","into","through", - "during","before","after","above","below","between","each","than","so", - "also","not","no","nor","yet","both","either","neither","just","because", - "if","then","else","when","where","how","all","any","both","few","more", - "most","other","some","such","up","out","about","per","which","their", - "our","your","my","his","her","they","we","you","he","she","i","me","us", - "him","them","what","who","whom","whose","there","here","now","then", -} - -def _tokenize(text: str) -> List[str]: - tokens = re.findall(r"[a-zA-Z]{3,}", text.lower()) - return [t for t in tokens if t not in _STOPWORDS] - -def extract_keywords(chunks: List[str], top_n: int = 10) -> List[str]: - """ - Lightweight TF-IDF keyword extractor. - chunks: list of raw text strings (already produced by chunker.py) - Returns top_n keywords sorted by TF-IDF score. - """ - if not chunks: - return [] - - # term frequency across the whole document - all_tokens = [] - doc_token_sets = [] - for chunk in chunks: - tokens = _tokenize(chunk) - all_tokens.extend(tokens) - doc_token_sets.append(set(tokens)) - - tf = Counter(all_tokens) - total = sum(tf.values()) or 1 - - n_docs = len(chunks) - scores: dict[str, float] = {} - for term, count in tf.items(): - tf_score = count / total - # document frequency = how many chunks contain this term - df = sum(1 for s in doc_token_sets if term in s) - idf = math.log((n_docs + 1) / (df + 1)) + 1 - scores[term] = tf_score * idf - - top = sorted(scores, key=lambda t: scores[t], reverse=True)[:top_n] - return top diff --git a/backend/app/rag/ocr.py b/backend/app/rag/ocr.py deleted file mode 100644 index 3e88a2f6..00000000 --- a/backend/app/rag/ocr.py +++ /dev/null @@ -1,180 +0,0 @@ -""" -OCR fallback for image-based PDF pages. - -Uses pytesseract (Tesseract OCR) to extract text from pages that contain -no selectable text. easyocr is supported as an optional alternative backend -controlled by the OCR_BACKEND environment variable. - -Detection strategy: a page is considered image-only when PyMuPDF's -get_text() returns fewer than MIN_TEXT_CHARS characters after stripping. -""" - -import logging -import os -from typing import Any, Dict, List - -import fitz # PyMuPDF - -logger = logging.getLogger(__name__) - -# Pages with fewer than this many characters are treated as image-only -MIN_TEXT_CHARS = 20 - -# OCR backend: "tesseract" (default) or "easyocr" -OCR_BACKEND = os.getenv("OCR_BACKEND", "tesseract").lower() - - -def _page_is_image_only(page: fitz.Page) -> bool: - """Return True when a PDF page has no meaningful selectable text.""" - text = page.get_text().strip() - return len(text) < MIN_TEXT_CHARS - - -def _render_page_to_image(page: fitz.Page, dpi: int = 200) -> bytes: - """Render a fitz page to PNG bytes at the given DPI.""" - mat = fitz.Matrix(dpi / 72, dpi / 72) - pix = page.get_pixmap(matrix=mat, alpha=False) - return pix.tobytes("png") - - -def _ocr_with_tesseract(image_bytes: bytes) -> str: - """Run Tesseract OCR on raw PNG bytes and return extracted text.""" - try: - import io - import pytesseract - from PIL import Image - except ImportError as exc: - raise ImportError( - "pytesseract and Pillow are required for OCR. " - "Install them with: pip install pytesseract Pillow" - ) from exc - - image = Image.open(io.BytesIO(image_bytes)) - text = pytesseract.image_to_string(image, lang="eng") - return text.strip() - - -def _ocr_with_easyocr(image_bytes: bytes) -> str: - """Run EasyOCR on raw PNG bytes and return extracted text.""" - try: - import easyocr - import numpy as np - from PIL import Image - import io - except ImportError as exc: - raise ImportError( - "easyocr, Pillow, and numpy are required for EasyOCR. " - "Install them with: pip install easyocr Pillow numpy" - ) from exc - - # EasyOCR reader is expensive to initialise — cache it at module level - global _easyocr_reader # noqa: PLW0603 - if "_easyocr_reader" not in globals(): - _easyocr_reader = easyocr.Reader(["en"], gpu=False, verbose=False) - - image = Image.open(io.BytesIO(image_bytes)).convert("RGB") - img_array = np.array(image) - results = _easyocr_reader.readtext(img_array, detail=0, paragraph=True) - return "\n".join(results).strip() - - -def ocr_page(page: fitz.Page, dpi: int = 200) -> str: - """ - OCR a single fitz page and return the extracted text. - - Renders the page to a raster image at *dpi* resolution, then runs the - configured OCR backend (``OCR_BACKEND`` env var, default ``tesseract``). - - Args: - page: An open fitz.Page object. - dpi: Render resolution. Higher values give better accuracy at the - cost of memory and speed. - - Returns: - Extracted text string, possibly empty if OCR yields nothing. - - Raises: - ImportError: If the selected backend's dependencies are not installed. - """ - image_bytes = _render_page_to_image(page, dpi=dpi) - - if OCR_BACKEND == "easyocr": - return _ocr_with_easyocr(image_bytes) - return _ocr_with_tesseract(image_bytes) - - -def extract_pdf_with_ocr(filepath: str, dpi: int = 200) -> List[Dict[str, Any]]: - """ - Extract text from a PDF, using OCR for image-only pages. - - For each page: - - If PyMuPDF can extract at least ``MIN_TEXT_CHARS`` characters, - use that text directly (fast path). - - Otherwise render the page and run OCR (slow path). - - Pages that yield no text via either method are skipped. - - Args: - filepath: Absolute path to the PDF file. - dpi: Render DPI for OCR pages (default 200). - - Returns: - List of dicts with keys ``text``, ``page``, ``chunk_type``, - and ``ocr`` (bool, True when the text came from OCR). - """ - doc = fitz.open(filepath) - pages: List[Dict[str, Any]] = [] - - try: - for page_num, page in enumerate(doc, start=1): - native_text = page.get_text().strip() - - if len(native_text) >= MIN_TEXT_CHARS: - pages.append({ - "text": native_text, - "page": page_num, - "chunk_type": "text", - "ocr": False, - }) - continue - - # Image-only page — fall back to OCR - logger.info( - "Page %d of '%s' has no selectable text — running OCR (backend=%s)", - page_num, - filepath, - OCR_BACKEND, - ) - try: - ocr_text = ocr_page(page, dpi=dpi) - if ocr_text: - pages.append({ - "text": ocr_text, - "page": page_num, - "chunk_type": "text", - "ocr": True, - }) - else: - logger.warning( - "OCR returned no text for page %d of '%s'", - page_num, - filepath, - ) - except ImportError: - logger.warning( - "OCR backend '%s' not installed — skipping page %d of '%s'", - OCR_BACKEND, - page_num, - filepath, - ) - except Exception as exc: - logger.warning( - "OCR failed for page %d of '%s': %s", - page_num, - filepath, - exc, - ) - finally: - doc.close() - - return pages diff --git a/backend/app/rag/prompts.py b/backend/app/rag/prompts.py index d5250c90..42bcc31e 100644 --- a/backend/app/rag/prompts.py +++ b/backend/app/rag/prompts.py @@ -87,11 +87,3 @@ {chat_history} Question: {input} Thought: {agent_scratchpad}""" - -MULTI_DOC_COMPARISON_GUIDANCE = """ -MULTI-DOCUMENT MODE: -You are answering across multiple documents at once. When findings differ or overlap between documents: -- Attribute each finding to its specific source document using [Source: filename, Page X]. -- Explicitly note where documents agree and where they disagree or report different figures. -- Do not blend numbers or claims from different documents without making the source of each clear. -""" \ No newline at end of file diff --git a/backend/app/rag/reranker.py b/backend/app/rag/reranker.py index f43116ba..ba5adefc 100644 --- a/backend/app/rag/reranker.py +++ b/backend/app/rag/reranker.py @@ -67,10 +67,10 @@ def rerank( model = self._load_model() - # Prepare query-document pairs - pairs = [(query, doc[text_key]) for doc in documents] + # Safely extract text, fallback to an empty string if key is missing to avoid crashes + pairs = [(query, doc.get(text_key, "")) for doc in documents] - # Get relevance scores + # Get relevance scores (utilizing batch_size to prevent OOM errors) scores = model.predict(pairs) # Pair scores with documents and sort in descending order diff --git a/backend/app/rag/retriever.py b/backend/app/rag/retriever.py index cf16c32f..e542c17f 100644 --- a/backend/app/rag/retriever.py +++ b/backend/app/rag/retriever.py @@ -1,11 +1,32 @@ """ -Two-stage retrieval: Hybrid Search (Vector + BM25 via RRF) + cross-encoder reranking. +Two-stage retrieval: Hybrid Ensemble (ChromaDB + BM25) + cross-encoder reranking. """ import json import logging import re from typing import List, Dict, Any, Optional +try: + # In LangChain 1.3.2+, EnsembleRetriever moved to langchain_classic. + from langchain_classic.retrievers import EnsembleRetriever +except ImportError: + class EnsembleRetriever: + """Small fallback used when optional LangChain classic deps are absent.""" + + def __init__(self, retrievers, weights=None): + self.retrievers = retrievers + self.weights = weights or [1.0] * len(retrievers) + + def invoke(self, query): + docs = [] + for retriever in self.retrievers: + docs.extend(retriever.invoke(query)) + return docs +from langchain_core.retrievers import BaseRetriever +from langchain_core.documents import Document as LangchainDocument +from langchain_core.callbacks import CallbackManagerForRetrieverRun +from pydantic import Field + from app.config import get_settings from app.rag.embeddings import embed_query from app.rag.tracing import trace_function @@ -17,62 +38,41 @@ MAX_QUERY_VARIANTS = 4 -# ── RRF core ───────────────────────────────────────────────────────────────── - -def rrf_merge( - vector_results: List[Dict[str, Any]], - bm25_results: List[Dict[str, Any]], - k: int = 60, -) -> List[Dict[str, Any]]: - """Merge vector and BM25 ranked lists using Reciprocal Rank Fusion. - - RRF formula: score(d) = Σ 1 / (k + rank(d, list)) - where rank is 1-based and k=60 is the standard smoothing constant. - - Args: - vector_results: Chunks from ChromaDB, ordered by descending similarity. - bm25_results: Chunks from BM25, ordered by descending BM25 score. - k: RRF smoothing constant (default 60). +class CustomVectorRetriever(BaseRetriever): + user_id: str = Field(description="User ID") + document_id: Optional[str] = Field(default=None, description="Document ID") + top_k: int = Field(default=10, description="Top K results") - Returns: - Deduplicated list of chunks sorted by descending RRF score, each chunk - carrying an ``rrf_score`` field. - """ - rrf_scores: Dict[str, float] = {} - chunk_store: Dict[str, Dict[str, Any]] = {} - - def _key(chunk: Dict[str, Any]) -> str: - """Stable deduplication key — prefer explicit IDs, fall back to content hash.""" - for field in ("id", "chunk_id"): - if chunk.get(field): - return str(chunk[field]) - text = str(chunk.get("text", "")) - return "|".join([ - str(chunk.get("document_id", "")), - str(chunk.get("page", "")), - text[:200], - ]) - - def _accumulate(results: List[Dict[str, Any]]) -> None: - for rank, chunk in enumerate(results, start=1): - key = _key(chunk) - rrf_scores[key] = rrf_scores.get(key, 0.0) + 1.0 / (k + rank) - if key not in chunk_store or chunk.get("score", 0) > chunk_store[key].get("score", 0): - chunk_store[key] = chunk - - _accumulate(vector_results) - _accumulate(bm25_results) + def _get_relevant_documents( + self, query: str, *, run_manager: CallbackManagerForRetrieverRun + ) -> List[LangchainDocument]: + query_vector = embed_query(query) + candidates = query_chunks( + query_embedding=query_vector, + user_id=self.user_id, + document_id=self.document_id, + top_k=self.top_k, + ) + return [LangchainDocument(page_content=c["text"], metadata=c) for c in candidates] - merged = [] - for key, rrf_score in sorted(rrf_scores.items(), key=lambda t: t[1], reverse=True): - chunk = chunk_store[key].copy() - chunk["rrf_score"] = round(rrf_score, 6) - merged.append(chunk) - return merged +class CustomBM25Retriever(BaseRetriever): + user_id: str = Field(description="User ID") + document_id: Optional[str] = Field(default=None, description="Document ID") + top_k: int = Field(default=10, description="Top K results") + def _get_relevant_documents( + self, query: str, *, run_manager: CallbackManagerForRetrieverRun + ) -> List[LangchainDocument]: + from app.rag.bm25 import query_bm25 + candidates = query_bm25( + query=query, + user_id=self.user_id, + document_id=self.document_id, + top_k=self.top_k, + ) + return [LangchainDocument(page_content=c["text"], metadata=c) for c in candidates] -# ── Query helpers ───────────────────────────────────────────────────────────── def transform_query(query: str) -> List[str]: """Rewrite a user question into multiple retrieval-friendly search queries.""" @@ -83,61 +83,43 @@ def transform_query(query: str) -> List[str]: try: generated_queries = _generate_query_variants(original_query) except Exception as e: - logger.warning("Query transformation failed, using original query only: %s", e) + logger.warning(f"Query transformation failed, using original query only: {e}") generated_queries = [] return _dedupe_queries([original_query, *generated_queries])[:MAX_QUERY_VARIANTS] def _generate_query_variants(query: str) -> List[str]: - """Use the configured LLM to rewrite a user query into 3 semantic variations. - - Each variation rephrases the original from a different angle so that - BM25 and ChromaDB retrieve a broader, complementary set of chunks. - The original query is always prepended by the caller (transform_query), - so we ask for exactly 3 *additional* variants here. - """ + """Use the configured LLM to split/rewrite a user query for semantic search.""" if not settings.HF_TOKEN: return [] from huggingface_hub import InferenceClient client = InferenceClient(token=settings.HF_TOKEN) - prompt = ( - "Generate exactly 3 semantic variations of the user question below. " - "Each variation must preserve the original meaning but use different " - "vocabulary, phrasing, or sentence structure to improve document retrieval coverage. " - "Do NOT add new topics or change the intent. " - "Return ONLY a JSON array of 3 strings, with no extra text, markdown, or explanation.\n\n" - f"User question: {query}\n\n" - 'Example output: ["variation one", "variation two", "variation three"]' + "Rewrite the user question into concise semantic search queries for document retrieval. " + "Split independent topics into separate queries. Return a JSON array of strings only. " + f"User question: {query}" ) - response = client.chat_completion( messages=[ { "role": "system", - "content": ( - "You are a query rewriter for a RAG retrieval system. " - "You output only valid JSON arrays of strings, nothing else." - ), + "content": "You create optimized search queries for a RAG retriever.", }, {"role": "user", "content": prompt}, ], model=settings.LLM_MODEL, max_tokens=256, - temperature=0.3, + temperature=0.2, ) if not response.choices: return [] content = response.choices[0].message.content or "" - variants = _parse_query_variants(content) - - # Cap at 3 variants as requested — the original is added by transform_query - return variants[:3] + return _parse_query_variants(content) def _parse_query_variants(content: str) -> List[str]: @@ -191,116 +173,111 @@ def _dedupe_queries(queries: List[str]) -> List[str]: return deduped +def _candidate_key(chunk: Dict[str, Any]) -> str: + for key in ("id", "chunk_id"): + if chunk.get(key): + return str(chunk[key]) + + text = str(chunk.get("text", "")) + return "|".join( + str(part) + for part in ( + chunk.get("document_id", ""), + chunk.get("filename", ""), + chunk.get("page", ""), + text[:200], + ) + ) + + def _merge_candidates(candidates: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Deduplicate a flat candidate list, keeping the highest-scored entry per key.""" merged: Dict[str, Dict[str, Any]] = {} + for candidate in candidates: candidate_copy = dict(candidate) - key = "|".join([ - str(candidate_copy.get("document_id", "")), - str(candidate_copy.get("page", "")), - str(candidate_copy.get("text", ""))[:200], - ]) + key = _candidate_key(candidate_copy) existing = merged.get(key) + if existing is None or candidate_copy.get("score", 0) > existing.get("score", 0): merged[key] = candidate_copy - return list(merged.values()) + return list(merged.values()) -# ── Main retrieval pipeline ─────────────────────────────────────────────────── @trace_function( "retrieve", - metadata_factory=lambda query, user_id, document_id=None, document_ids=None, top_k=None: { + metadata_factory=lambda query, user_id, document_id=None, top_k=None: { "user_id": user_id, "document_id": document_id, "embedding_model": settings.EMBEDDING_MODEL, "reranker_model": settings.RERANKER_MODEL, "top_k_retrieval": settings.TOP_K_RETRIEVAL, "top_k_rerank": settings.TOP_K_RERANK, - "hybrid_search": settings.USE_HYBRID_SEARCH, - "rrf_k": settings.RRF_K, }, ) def retrieve( query: str, user_id: str, document_id: Optional[str] = None, - document_ids: Optional[List[str]] = None, top_k: Optional[int] = None, ) -> List[Dict[str, Any]]: """ Two-stage retrieval pipeline: - 1. Hybrid Search — Vector (ChromaDB) + BM25 merged via Reciprocal Rank Fusion (RRF), - applied across all transformed query variants. - 2. Cross-encoder reranking — top-K refined by a cross-encoder model. + 1. Hybrid Search (Vector + BM25 via EnsembleRetriever with RRF) with Query Transformation + 2. Cross-encoder reranking (top-K refined) - Falls back to vector-only when USE_HYBRID_SEARCH=False or rank_bm25 is absent. Returns chunks with confidence scores. """ + # ── Stage 1: Hybrid Search with Query Transformation ───────────── effective_top_k = top_k if top_k is not None else settings.TOP_K_RETRIEVAL + vector_retriever = CustomVectorRetriever( + user_id=user_id, + document_id=document_id, + top_k=effective_top_k, + ) - # ── Stage 1: Hybrid retrieval with query transformation ─────────────────── - all_candidates: List[Dict[str, Any]] = [] - - for search_query in transform_query(query): - query_vector = embed_query(search_query) + bm25_retriever = CustomBM25Retriever( + user_id=user_id, + document_id=document_id, + top_k=effective_top_k, + ) - # Vector results (always) - vector_results = query_chunks( - query_embedding=query_vector, - user_id=user_id, - document_id=document_id, - document_ids=document_ids, - top_k=effective_top_k, - ) + ensemble_retriever = EnsembleRetriever( + retrievers=[vector_retriever, bm25_retriever], + weights=[0.6, 0.4] + ) - if settings.USE_HYBRID_SEARCH: - try: - from app.rag.bm25 import query_bm25 - bm25_results = query_bm25( - query=search_query, - user_id=user_id, - document_id=document_id, - document_ids=document_ids, - top_k=effective_top_k, - ) - except Exception as exc: - logger.warning("BM25 retrieval failed, using vector-only: %s", exc) - bm25_results = [] - - merged = rrf_merge( - vector_results=vector_results, - bm25_results=bm25_results, - k=settings.RRF_K, - ) - - for chunk in merged: - chunk["score"] = chunk.pop("rrf_score") - - all_candidates.extend(merged) - else: - all_candidates.extend(vector_results) + all_candidates = [] + for search_query in transform_query(query): + docs = ensemble_retriever.invoke(search_query) + for i, doc in enumerate(docs): + chunk = doc.metadata.copy() + # Preserve a mock score based on rank for fallback if reranker fails + # We use 1.0/(i+1) as a base RRF-like score + chunk["score"] = 1.0 / (i + 1) + all_candidates.append(chunk) if not all_candidates: return [] candidates = _merge_candidates(all_candidates) - # ── Stage 2: Cross-encoder reranking ───────────────────────────────────── + # ── Stage 2: Cross-encoder reranking ───────────── reranker = get_reranker() - + if reranker is not None: top_chunks = reranker.rerank( query=query, documents=candidates, - top_k=settings.TOP_K_RERANK, + top_k=settings.TOP_K_RERANK ) else: + # Fall back to hybrid scores (no reranker) candidates.sort(key=lambda x: x.get("score", 0), reverse=True) top_chunks = candidates[:settings.TOP_K_RERANK] - # ── Confidence normalisation ────────────────────────────────────────────── + # top_chunks is now always defined + # ── Calculate confidence percentages ───────────── if top_chunks: max_score = max( chunk.get("rerank_score", chunk.get("score", 0)) @@ -315,12 +292,4 @@ def retrieve( chunk["score"] = round(chunk["rerank_score"], 4) del chunk["rerank_score"] - from app.observability import chunks_retrieved_var - chunks_retrieved_var.set(len(top_chunks)) - logger.info( - "Retrieved %d relevant chunks for query: '%s'", - len(top_chunks), - query, - ) - return top_chunks diff --git a/backend/app/rag/security.py b/backend/app/rag/security.py index b3b59d89..990f87fe 100644 --- a/backend/app/rag/security.py +++ b/backend/app/rag/security.py @@ -42,15 +42,9 @@ class InputClassification: reason: str | None = None -from app.exceptions import AppException - - -class UnsafePromptError(AppException): +class UnsafePromptError(ValueError): """Raised when user input matches prompt-injection patterns.""" - def __init__(self, message: str = BLOCKED_INPUT_MESSAGE): - super().__init__("UNSAFE_PROMPT", message, 400) - class OutputParserError(ValueError): """Raised when the LLM response does not match the required schema.""" diff --git a/backend/app/rag/summarizer.py b/backend/app/rag/summarizer.py index d21fd6d7..4310cf9c 100644 --- a/backend/app/rag/summarizer.py +++ b/backend/app/rag/summarizer.py @@ -7,11 +7,7 @@ logger = logging.getLogger(__name__) settings = get_settings() -def generate_document_summary( - filePath: str | None = None, - max_sentences: int = 3, - chunks: list[dict] | None = None -) -> str | None: +def generate_document_summary(filePath: str, max_sentences: int = 3) -> str | None: """ Extract text from the first few chunks of the document and ask LLM to summarise. Returns a short summary string, or None on failure. @@ -30,29 +26,17 @@ def generate_document_summary( from app.rag.chunker import chunk_document try: - # Fall back to file parsing only if chunks are not pre-extracted - if chunks is None: - if not filePath: - logger.error("Neither 'chunks' nor 'filePath' was provided.") - return None - chunks = chunk_document(filePath) + chunks = chunk_document(filePath) if not chunks: - identifier = filePath if filePath else "provided chunks" - logger.warning(f"No chunks available for {identifier}, cannot summarise.") + logger.warning(f"No chunks extracted from {filePath}, cannot summarise.") return None # Extract text from each chunk and concatenate for summarisation chunk_texts = [] for chunk in chunks[:10]: # Use first 10 chunks to limit input size text = chunk.get("text") - # Ensure text is explicitly a string instance and not just whitespace - if isinstance(text, str) and text.strip(): - chunk_texts.append(text) - - if not chunk_texts: - logger.warning("Extracted chunks contained no valid text content.") - return None + chunk_texts.append(text) text_to_summarise = " ".join(chunk_texts) @@ -66,19 +50,10 @@ def generate_document_summary( max_tokens=settings.SUMMARY_MAX_TOKENS, temperature=settings.LLM_TEMPERATURE, ) - - # Defensive check for malformed or empty response structures - summary = None - if response and getattr(response, "choices", None): - first_choice = response.choices[0] - message = getattr(first_choice, "message", None) - content = getattr(message, "content", None) - if content: - summary = content.strip() + summary = response.choices[0].message.content.strip() if response.choices else None - return summary or None + return summary if summary else None except Exception as e: - identifier = filePath if filePath else "pre-extracted chunks" - logger.error(f"Summary generation failed for {identifier}: {e}") + logger.error(f"Summary generation failed for {filePath}: {e}") return None \ No newline at end of file diff --git a/backend/app/rag/tools.py b/backend/app/rag/tools.py index 01542a9e..03813756 100644 --- a/backend/app/rag/tools.py +++ b/backend/app/rag/tools.py @@ -156,7 +156,6 @@ class PDFSearchTool(BaseTool): user_id: str document_id: Optional[str] = None - document_ids: Optional[List[str]] = None top_k: Optional[int] = None # We'll store sources here to retrieve them after agent execution last_sources: List[Dict[str, Any]] = [] @@ -168,7 +167,6 @@ def _run(self, query: str) -> str: query=query, user_id=self.user_id, document_id=self.document_id, - document_ids=self.document_ids, top_k=self.top_k, ) diff --git a/backend/app/rag/tracing.py b/backend/app/rag/tracing.py index 0f44299e..f95e8b18 100644 --- a/backend/app/rag/tracing.py +++ b/backend/app/rag/tracing.py @@ -87,17 +87,7 @@ def trace_function( def decorator(fn: Callable[..., Any]) -> Callable[..., Any]: @wraps(fn) def wrapped(*args: Any, **kwargs: Any) -> Any: - metadata = None - if metadata_factory: - try: - metadata = metadata_factory(*args, **kwargs) - except Exception: - logger.warning( - "Metadata factory failed for trace %r; continuing without metadata.", - name, - exc_info=True, - ) - metadata = {} + metadata = metadata_factory(*args, **kwargs) if metadata_factory else None return trace_call( name, fn, diff --git a/backend/app/rag/url_extractor.py b/backend/app/rag/url_extractor.py deleted file mode 100644 index 917e1fcf..00000000 --- a/backend/app/rag/url_extractor.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -PDF URL extraction using PyMuPDF link annotations. - -Extracts all unique HTTP/HTTPS URLs from a PDF's link annotations -and text-based URIs across all pages. Called during document ingestion -so URLs are stored in the document metadata column. -""" -import logging -import re -from typing import List - -logger = logging.getLogger(__name__) - -# Matches http/https URLs in plain text as a fallback -_URL_RE = re.compile( - r"https?://" - r"(?:[A-Za-z0-9\-._~:/?#\[\]@!$&'()*+,;=%])" - r"[A-Za-z0-9\-._~:/?#\[\]@!$&'()*+,;=%]*", - re.IGNORECASE, -) - - -def extract_urls_from_pdf(filepath: str) -> List[str]: - """Extract unique HTTP/HTTPS URLs from a PDF file. - - Two passes per page: - 1. Link annotations — catches hyperlinks embedded by the PDF author. - 2. Plain-text regex — catches URLs typed inline that aren't annotated. - - Args: - filepath: Absolute path to the PDF file. - - Returns: - Deduplicated list of URLs preserving first-seen order. - Returns an empty list for non-PDF files or on any extraction error. - """ - if not filepath.rsplit(".", 1)[-1].lower() == "pdf": - return [] - - try: - import fitz # PyMuPDF — already in requirements.txt - except ImportError: - logger.warning("PyMuPDF not available; skipping URL extraction") - return [] - - seen: dict = {} # preserves insertion order, deduplicates - doc = None - - try: - doc = fitz.open(filepath) - - for page_num, page in enumerate(doc): - # ── Pass 1: link annotations ────────────────────────────────── - for link in page.get_links(): - uri = link.get("uri", "") - if uri and uri.lower().startswith(("http://", "https://")): - seen.setdefault(uri.strip(), None) - - # ── Pass 2: plain-text regex ────────────────────────────────── - text = page.get_text() - for match in _URL_RE.finditer(text): - url = match.group(0).rstrip(".,;:)\"'") - seen.setdefault(url, None) - - except Exception as exc: - logger.warning("URL extraction failed for %s: %s", filepath, exc) - return [] - finally: - if doc: - doc.close() - - return list(seen.keys()) diff --git a/backend/app/rag/vectorstore.py b/backend/app/rag/vectorstore.py index 06ff44b8..07a68066 100644 --- a/backend/app/rag/vectorstore.py +++ b/backend/app/rag/vectorstore.py @@ -161,7 +161,7 @@ def query_chunks( query_embeddings=[query_embedding], n_results=top_k, where=where_filter, - include=["documents", "metadatas", "distances", "ids"], + include=["documents", "metadatas", "distances"], ) # ── Format results ─────────────────────────────── @@ -170,13 +170,11 @@ def query_chunks( for i, doc in enumerate(results["documents"][0]): metadata = results["metadatas"][0][i] if results["metadatas"] else {} distance = results["distances"][0][i] if results["distances"] else 0 - chunk_id = results["ids"][0][i] if results.get("ids") and len(results["ids"]) > 0 else None # Convert cosine distance to similarity score (0-1) similarity = 1 - distance chunks.append({ - "id": chunk_id, "text": doc, "filename": metadata.get("filename", ""), "document_id": metadata.get("document_id", ""), diff --git a/backend/app/rag/vision.py b/backend/app/rag/vision.py index 57068452..a84390d5 100644 --- a/backend/app/rag/vision.py +++ b/backend/app/rag/vision.py @@ -1,321 +1,99 @@ """Image captioning / vision helpers for RAG pipeline. -Caption resolution order for each image chunk: -1. Bounding-box proximity — nearest text block below/above the image in the PDF - (rich, zero-cost, works offline). -2. OCR (pytesseract) — when proximity yields nothing and tesseract is installed. -3. Placeholder — "Figure on page N (WxH px)" as a guaranteed non-empty fallback. - -An optional OpenAI GPT-4o-mini vision hook is provided for deployments that set -VISION_PROVIDER=openai and OPENAI_API_KEY in settings. +Provides a simple, pluggable interface to generate textual descriptions +for images extracted from PDFs. By default it uses local OCR (pytesseract) +when available as a robust fallback. An external VLM provider (OpenAI) +can be integrated by setting `VISION_PROVIDER` and appropriate API keys +in settings; the provider hook is intentionally small and optional. """ -import base64 import logging +from typing import List, Dict, Any from io import BytesIO -from typing import Any, Dict, List, Optional - -import fitz # PyMuPDF from app.config import get_settings logger = logging.getLogger(__name__) settings = get_settings() -# ── Optional OCR backend (PIL + pytesseract) ───────────────────────────────── -# Imported once at module load instead of inline on every _ocr_caption() call. -# ``HAS_OCR`` records availability so the hot path (large batch caption loops in -# generate_captions_for_chunks) can short-circuit with a cheap boolean check -# rather than re-running an import + try/except on each image. -try: - from PIL import Image - import pytesseract - - HAS_OCR = True -except ImportError: - Image = None # type: ignore[assignment] - pytesseract = None # type: ignore[assignment] - HAS_OCR = False - logger.info( - "OCR backend unavailable (PIL/pytesseract not installed); " - "image captioning will fall back to placeholders." - ) - -# Minimum image area (px²) — smaller images are decorative and skipped. -_MIN_IMAGE_AREA = 1_000 - - -# ── 1. Proximity-based caption extraction ──────────────────────────────────── - -def _find_caption_near_image( - page: fitz.Page, - img_bbox: fitz.Rect, - search_margin: float = 60.0, -) -> str: - """Return the closest text block directly below (or above) an image rect.""" - page_dict = page.get_text("dict", flags=fitz.TEXT_PRESERVE_WHITESPACE) - blocks = page_dict.get("blocks", []) - - def _closest(region: fitz.Rect) -> str: - candidates = [] - for block in blocks: - if block.get("type") != 0: # 0 == text block - continue - bx0, by0, bx1, by1 = block["bbox"] - if fitz.Rect(bx0, by0, bx1, by1).intersects(region): - text = " ".join( - span["text"] - for line in block.get("lines", []) - for span in line.get("spans", []) - ).strip() - if text: - candidates.append((abs(by0 - img_bbox.y1), text)) - if candidates: - return min(candidates, key=lambda t: t[0])[1] - return "" - - # Search below first, fall back to above - below = fitz.Rect(img_bbox.x0, img_bbox.y1, img_bbox.x1, img_bbox.y1 + search_margin) - caption = _closest(below) - if caption: - return caption - - above = fitz.Rect(img_bbox.x0, img_bbox.y0 - search_margin, img_bbox.x1, img_bbox.y0) - return _closest(above) - - -def extract_captions_from_pdf(filepath: str) -> List[Dict[str, Any]]: - """Extract proximity-based image captions from a PDF. - - Returns a list of dicts ordered by (page, figure_index): - { - "page": int, # 1-based - "figure_index": int, # 0-based within the page - "caption": str, # may be empty string - "bbox": list[float], # [x0, y0, x1, y1] normalised to [0, 1] - } - """ - results: List[Dict[str, Any]] = [] - doc = fitz.open(filepath) - - try: - for page_num, page in enumerate(doc): - W, H = float(page.rect.width), float(page.rect.height) - figure_index = 0 - - for img_info in page.get_images(full=True): - xref = img_info[0] - try: - rects = page.get_image_rects(xref) - if not rects: - continue - img_rect = rects[0] - - if img_rect.width * img_rect.height < _MIN_IMAGE_AREA: - continue # skip decorative images - - caption = _find_caption_near_image(page, img_rect) - results.append( - { - "page": page_num + 1, - "figure_index": figure_index, - "caption": caption, - "bbox": [ - round(img_rect.x0 / W, 4), - round(img_rect.y0 / H, 4), - round(img_rect.x1 / W, 4), - round(img_rect.y1 / H, 4), - ], - } - ) - figure_index += 1 - - except Exception as exc: - logger.warning( - "Skipping image xref=%s on page %s: %s", xref, page_num + 1, exc - ) - finally: - doc.close() - - return results - - -# ── 2. OCR fallback ────────────────────────────────────────────────────────── def _ocr_caption(image_bytes: bytes) -> str: - """Attempt OCR via pytesseract; returns empty string if unavailable. - - The PIL/pytesseract import is resolved once at module load (see ``HAS_OCR``), - so this only does a boolean check before touching the image bytes. - """ - if not HAS_OCR: - return "" - + """Try to produce a caption using pytesseract OCR; returns empty string if not available.""" try: - img = Image.open(BytesIO(image_bytes)).convert("RGB") - text = pytesseract.image_to_string(img).strip() - return (text[:500] + "...") if len(text) > 500 else text - except Exception as exc: - logger.debug("OCR failed: %s", exc) - return "" - - -# ── 3. Optional OpenAI GPT-4o-mini vision hook ─────────────────────────────── - -def _openai_caption(image_bytes: bytes) -> str: - """Call OpenAI Chat Completions vision API; returns empty string on any failure.""" - api_key = getattr(settings, "OPENAI_API_KEY", None) - if not api_key: + from PIL import Image + import pytesseract + except Exception: return "" try: - from openai import OpenAI - - client = OpenAI(api_key=api_key) - b64 = base64.b64encode(image_bytes).decode("utf-8") - - response = client.chat.completions.create( - model="gpt-4o-mini", - max_tokens=120, - messages=[ - { - "role": "user", - "content": [ - { - "type": "image_url", - "image_url": { - "url": f"data:image/png;base64,{b64}", - "detail": "low", - }, - }, - { - "type": "text", - "text": ( - "Describe this figure or diagram in one concise sentence " - "suitable for use as a search index caption." - ), - }, - ], - } - ], - ) - return response.choices[0].message.content.strip() - - except Exception as exc: - logger.debug("OpenAI vision caption failed: %s", exc) + img = Image.open(BytesIO(image_bytes)).convert("RGB") + text = pytesseract.image_to_string(img) + text = text.strip() + return text + except Exception as e: + logger.debug(f"OCR failed: {e}") return "" -# ── Public API ─────────────────────────────────────────────────────────────── - -def caption_image(image_bytes: bytes, page: Optional[int] = None) -> str: - """Generate a caption for a single image (bytes). - - Resolution order: OpenAI (if configured) → OCR → placeholder. - """ -def caption_image(image_bytes: bytes | List[bytes], page: int | List[int] | None = None) -> str | List[str]: - """Generate a caption for a single image or a batch of images. +def caption_image(image_bytes: bytes, page: int | None = None) -> str: + """Generate a caption for a single image. Order of operations: - - If a list of image bytes is passed, returns a list of captions. - - If an external VLM provider is configured, attempt to call it. + - If an external VLM provider is configured, attempt to call it (not implemented as mandatory). - Fall back to local OCR (pytesseract) if available. - Otherwise return a simple placeholder caption including the page number. """ - if isinstance(image_bytes, list): - pages = page if isinstance(page, list) else ([page] * len(image_bytes) if page is not None else [None] * len(image_bytes)) - return [caption_image(img, pg) for img, pg in zip(image_bytes, pages)] - # Placeholder for provider-based captioning (e.g., OpenAI / LLaVA hooks) provider = getattr(settings, "VISION_PROVIDER", None) - if provider == "openai": try: - import base64 - from openai import OpenAI - + import openai + # Minimal integration: attempt a text-only caption via responses if available. + # This is a best-effort hook; users should adapt to their provider's API. api_key = getattr(settings, "OPENAI_API_KEY", None) if api_key: - # Initialize modern client - client = OpenAI(api_key=api_key) - - # Base64 encode the incoming image bytes - base64_image = base64.b64encode(image_bytes).decode('utf-8') - - # Request a visual caption using Chat Completions payload structure - resp = client.chat.completions.create( - model="gpt-4o-mini", - messages=[ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Describe this image in one concise sentence." - }, - { - "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{base64_image}" - } - } - ] - } - ], - max_tokens=150 + openai.api_key = api_key + # Use a generic prompt: "Describe the following image" + # Note: concrete multimodal API usage may vary across SDK versions. + resp = openai.Image.create( + prompt="Describe this image in one concise sentence.", + n=1, + # We do not re-upload image bytes here; this is a placeholder to show + # where provider code would be invoked. For production, follow + # provider docs for sending image data. ) - - # Extract and return the caption immediately if successful - caption_text = resp.choices[0].message.content - if caption_text: - return caption_text.strip() - - except Exception as e: - # Enhanced error logging to make debugging transparent - logger.warning(f"OpenAI vision provider failed: {e}, falling back to OCR") + # openai.Image.create returns generated images, not captions — so skip. + except Exception: + # If provider integration fails, fall back to OCR below + logger.debug("OpenAI vision provider failed, falling back to OCR") # Try OCR caption ocr = _ocr_caption(image_bytes) if ocr: - return ocr + # Keep it short if very long + return (ocr[:500] + "...") if len(ocr) > 500 else ocr - # Derive dimensions for the placeholder - try: - pix = fitz.Pixmap(image_bytes) - dims = f"{pix.width}x{pix.height} px" - except Exception: - dims = "unknown size" - - return f"Figure on page {page} ({dims})." if page else f"Figure ({dims})." + # Last-resort caption + if page: + return f"Image on page {page}." + return "Image." def generate_captions_for_chunks(chunks: List[Dict[str, Any]]) -> None: - """Mutate image chunks in-place: fill empty ``text`` with a caption. - - Called by vectorstore.store_chunks() before embedding. - Proximity-based captions should already be written into chunk["image_caption"] - by document_ingestion.ingest_document() before this point. - This function handles the OCR / placeholder fallback for any remaining gaps. + """Mutate chunks in-place: for any chunk containing `image_bytes` but empty `text`, + generate a caption and set `text`. """ for chunk in chunks: - if not chunk.get("image_bytes"): - continue - if chunk.get("text", "").strip(): - continue # already captioned by proximity pass - - try: - # Use pre-extracted proximity caption if available - caption = chunk.get("image_caption") or caption_image( - chunk["image_bytes"], page=chunk.get("page") - ) - chunk["text"] = caption - chunk["is_image"] = True - chunk["image_caption"] = caption - except Exception as exc: - logger.debug("Failed to caption image chunk: %s", exc) - chunk["is_image"] = True - fallback = f"Image on page {chunk.get('page', '?')}" - chunk.setdefault("text", fallback) - chunk["image_caption"] = chunk["text"] - finally: - # Always strip raw bytes — never serialise them into ChromaDB - chunk.pop("image_bytes", None) + if chunk.get("image_bytes") and not chunk.get("text"): + try: + caption = caption_image(chunk["image_bytes"], page=chunk.get("page")) + chunk["text"] = caption + # Remove raw bytes to avoid accidentally serializing them later + chunk.pop("image_bytes", None) + chunk["is_image"] = True + chunk["image_caption"] = caption + except Exception as e: + logger.debug(f"Failed to caption image chunk: {e}") + # ensure we still mark it as image to avoid losing it + chunk.pop("image_bytes", None) + chunk["is_image"] = True + chunk.setdefault("text", f"Image on page {chunk.get('page')}") diff --git a/backend/app/rate_limit.py b/backend/app/rate_limit.py index d7759de2..651c5aab 100644 --- a/backend/app/rate_limit.py +++ b/backend/app/rate_limit.py @@ -2,16 +2,13 @@ SlowAPI rate limiting configuration. """ from fastapi import Request -from limits import parse -from limits.storage import storage_from_string -from limits.strategies import FixedWindowRateLimiter from slowapi import Limiter from slowapi.util import get_remote_address - - + + CHAT_QUERY_RATE_LIMIT = "15/minute" - - + + def rate_limit_key_func(request: Request) -> str: """Use authenticated user id when available, otherwise fall back to client IP.""" authorization = request.headers.get("authorization", "") @@ -23,32 +20,8 @@ def rate_limit_key_func(request: Request) -> str: if user_id: return f"user:{user_id}" except Exception: - pass + pass return f"ip:{get_remote_address(request)}" limiter = Limiter(key_func=rate_limit_key_func) - - -# SlowAPI's @limiter.limit decorator only wraps standard HTTP route handlers -# (e.g. POST /chat/ask, /chat/ask/stream in app/routes/chat.py) — it has no -# WebSocket support, so it can't be applied to the /chat/ws endpoint. This -# dedicated strategy + in-memory storage backs a manual check that enforces -# the same CHAT_QUERY_RATE_LIMIT, keyed the same way ("user:") that -# rate_limit_key_func uses for the HTTP routes. -_chat_ws_rate_limit_item = parse(CHAT_QUERY_RATE_LIMIT) -_chat_ws_storage = storage_from_string("memory://") -_chat_ws_limiter = FixedWindowRateLimiter(_chat_ws_storage) - - -def check_chat_ws_rate_limit(user_id: str) -> bool: - """Consume one hit against the /chat/ws rate-limit bucket for a user. - - Mirrors CHAT_QUERY_RATE_LIMIT (15/minute), which @limiter.limit enforces - on /chat/ask and /chat/ask/stream, for the WebSocket transport that the - SlowAPI decorator can't reach. - - Returns True if the request is within the limit, False if the caller has - exceeded it and the request should be rejected. - """ - return _chat_ws_limiter.hit(_chat_ws_rate_limit_item, f"user:{user_id}") \ No newline at end of file diff --git a/backend/app/routes/admin.py b/backend/app/routes/admin.py index 5490d845..2b0fc817 100644 --- a/backend/app/routes/admin.py +++ b/backend/app/routes/admin.py @@ -1,14 +1,12 @@ """ -Admin-only operational statistics and database maintenance routes. +Admin-only operational statistics routes. """ -import json import shutil -from datetime import datetime, timezone, date from pathlib import Path from typing import List -from fastapi import APIRouter, Depends, HTTPException, Response -from sqlalchemy import func, text, inspect +from fastapi import APIRouter, Depends +from sqlalchemy import func from sqlalchemy.orm import Session from app.auth import get_current_admin @@ -49,7 +47,12 @@ def get_admin_stats( db: Session = Depends(get_db), _admin: User = Depends(get_current_admin), ): - """Return aggregate operational statistics for the admin dashboard.""" + """Return aggregate operational statistics for the admin dashboard. + + The response includes counts for users, uploaded PDFs, all documents, chat + messages, average RAG query latency, and upload-directory disk usage. + Access is restricted by the `get_current_admin` dependency. + """ upload_dir = Path(settings.UPLOAD_DIR).resolve() upload_dir.mkdir(parents=True, exist_ok=True) @@ -97,93 +100,9 @@ def list_all_users( db: Session = Depends(get_db), _admin: User = Depends(get_current_admin), ): - """List all registered users.""" - return db.query(User).all() + """List all registered users. - -@router.get( - "/export-db", - summary="Export database backup", - description="Dumps all database tables securely into either a JSON document or a SQL injection script.", -) -def export_database( - format: str = "json", - db: Session = Depends(get_db), - _admin: User = Depends(get_current_admin), -): - """Securely export all database table records for offsite backup support.""" - format_type = format.lower() - if format_type not in ["json", "sql"]: - raise HTTPException( - status_code=400, - detail="Invalid export format specified. Supported variants: json, sql" - ) - - inspector = inspect(db.get_bind()) - table_names = inspector.get_table_names() - timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") - - if format_type == "json": - backup_data = {} - for table in table_names: - result = db.execute(text(f"SELECT * FROM {table}")) - columns = list(result.keys()) - rows = [] - for row in result.fetchall(): - row_dict = {} - for col, val in zip(columns, row): - if isinstance(val, (datetime, date)): # ◄ Fixed type tuple evaluation - row_dict[col] = val.isoformat() - elif isinstance(val, bytes): - row_dict[col] = val.decode("utf-8", errors="ignore") - else: - row_dict[col] = val - rows.append(row_dict) - backup_data[table] = rows - - content = json.dumps(backup_data, indent=2, default=str) - filename = f"db_backup_{timestamp}.json" - media_type = "application/json" - - else: - sql_lines = [ - "-- Enterprise Agentic RAG System Database Backup", - f"-- Generated at: {datetime.now(timezone.utc).isoformat()}", - "-- Format: cross-compatible SQL script\n" - ] - for table in table_names: - result = db.execute(text(f"SELECT * FROM {table}")) - columns = list(result.keys()) - rows = result.fetchall() - if rows: - sql_lines.append(f"-- Data records for table: {table}") - cols_str = ", ".join([f'"{c}"' for c in columns]) - for row in rows: - vals = [] - for val in row: - if val is None: - vals.append("NULL") - elif isinstance(val, (int, float)): - vals.append(str(val)) - elif isinstance(val, bool): - vals.append("1" if val else "0") - elif isinstance(val, (datetime, date)): # ◄ Fixed type tuple evaluation - vals.append(f"'{val.isoformat()}'") - else: - escaped_val = str(val).replace("'", "''") - vals.append(f"'{escaped_val}'") - vals_str = ", ".join(vals) - sql_lines.append(f'INSERT INTO "{table}" ({cols_str}) VALUES ({vals_str});') - sql_lines.append("") - - content = "\n".join(sql_lines) - filename = f"db_backup_{timestamp}.sql" - media_type = "application/sql" - - headers = { - "Content-Disposition": f"attachment; filename={filename}", - "X-Content-Type-Options": "nosniff", - "Cache-Control": "no-store, no-cache, must-revalidate, max-age=0", - "Pragma": "no-cache", - } - return Response(content=content, media_type=media_type, headers=headers) + Access is restricted to administrators and the response is serialized + through `UserResponse` so token fields and secrets are not exposed. + """ + return db.query(User).all() diff --git a/backend/app/routes/auth.py b/backend/app/routes/auth.py index c8caaf58..e5345d93 100644 --- a/backend/app/routes/auth.py +++ b/backend/app/routes/auth.py @@ -11,7 +11,7 @@ import httpx import jwt -from fastapi import APIRouter, Depends, Query, status, Cookie, Response, Body +from fastapi import APIRouter, Depends, HTTPException, Query, status, Cookie, Response, Body from fastapi.responses import HTMLResponse, RedirectResponse from fastapi_mail import ConnectionConfig, FastMail, MessageSchema, MessageType from google_auth_oauthlib.flow import Flow @@ -21,14 +21,6 @@ from sqlalchemy import select from app.config import get_settings from app.database import get_db -from app.exceptions import ( - AppException, - ConflictException, - ExternalServiceException, - NotFoundException, - UnauthorizedException, - ValidationException, -) from app.models import User, ApiKey, UserRole from app.schemas import ( GoogleLoginRequest, @@ -39,7 +31,6 @@ TokenResponse, UpdatePassword, UpdatePasswordResponse, - ChangePasswordRequest, UserLogin, UserRegister, UserResponse, @@ -57,8 +48,6 @@ settings = get_settings() logger = logging.getLogger(__name__) -from app.audit import audit_log - REGISTRATION_MESSAGE = "Registration successful. Please check your email to verify your account before logging in." VERIFICATION_RESEND_MESSAGE = "If the email is registered and unverified, a verification link has been sent." VERIFICATION_REQUIRED_MESSAGE = "Please verify your email before logging in" @@ -139,13 +128,19 @@ def _create_token_response(user: User) -> TokenResponse: def _verify_google_token(id_token_value: str) -> dict: if not settings.GOOGLE_CLIENT_ID: - raise ExternalServiceException("Google", "Google sign-in is not configured") + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Google sign-in is not configured", + ) try: from google.auth.transport.requests import Request from google.oauth2 import id_token except ImportError as exc: - raise ExternalServiceException("Google", "Google authentication dependency is not installed") from exc + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Google authentication dependency is not installed", + ) from exc try: google_payload = id_token.verify_oauth2_token( @@ -154,11 +149,17 @@ def _verify_google_token(id_token_value: str) -> dict: settings.GOOGLE_CLIENT_ID, ) except ValueError as exc: - raise UnauthorizedException("Invalid Google credential") from exc + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid Google credential", + ) from exc email = google_payload.get("email") if not email or not google_payload.get("email_verified"): - raise UnauthorizedException("Google account email is not verified") + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Google account email is not verified", + ) return google_payload @@ -201,11 +202,17 @@ async def register(payload: UserRegister, db: Session = Depends(get_db)): """ # Check existing username if db.query(User).filter(User.username == payload.username).first(): - raise ConflictException("Username already taken") + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Username already taken", + ) # Check existing email if db.query(User).filter(User.email == payload.email).first(): - raise ConflictException("Email already registered") + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Email already registered", + ) user = User( username=payload.username, @@ -221,13 +228,6 @@ async def register(payload: UserRegister, db: Session = Depends(get_db)): await _send_verification_email(user, token) - audit_log( - action="user.register", - user_id=str(user.id), - result="success", - details={"email": user.email}, - ) - return RegistrationResponse( message=REGISTRATION_MESSAGE, email=user.email, @@ -261,33 +261,21 @@ def login(payload: UserLogin, db: Session = Depends(get_db)): user = db.query(User).filter(User.email == payload.email).first() if not user or not verify_password(payload.password, user.hashed_password): - audit_log( - action="user.login", - result="failure", - details={"email": payload.email, "reason": "invalid_credentials"}, + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid email or password", ) - raise UnauthorizedException("Invalid email or password") if not user.is_verified: - audit_log( - action="user.login", - user_id=str(user.id), - result="failure", - details={"reason": "email_not_verified"}, + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=VERIFICATION_REQUIRED_MESSAGE, ) - raise AppException("FORBIDDEN", VERIFICATION_REQUIRED_MESSAGE, 403) user.last_login = datetime.now(timezone.utc) db.commit() db.refresh(user) - audit_log( - action="user.login", - user_id=str(user.id), - result="success", - details={"method": "password"}, - ) - return _create_token_response(user) @@ -332,7 +320,10 @@ def verify_email(token: str, db: Session = Depends(get_db)): token_hash = _hash_verification_token(token) user = db.query(User).filter(User.verification_token_hash == token_hash).first() if not user: - raise ValidationException("Invalid or expired verification token") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid or expired verification token", + ) created_at = user.verification_token_created_at if created_at and created_at.tzinfo is None: @@ -344,7 +335,10 @@ def verify_email(token: str, db: Session = Depends(get_db)): user.verification_token_hash = None user.verification_token_created_at = None db.commit() - raise ValidationException("Invalid or expired verification token") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid or expired verification token", + ) user.is_verified = True user.verification_token_hash = None @@ -393,11 +387,17 @@ def refresh_token(payload: RefreshRequest, db: Session = Depends(get_db)): """ user_id = decode_token(payload.refresh_token, token_type="refresh") if not user_id: - raise UnauthorizedException("Invalid or expired refresh token") + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or expired refresh token", + ) user = db.query(User).filter(User.id == user_id).first() if not user: - raise UnauthorizedException("User not found") + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="User not found", + ) return _create_token_response(user) @@ -486,27 +486,30 @@ def update_user_info(payload:UserUpdate, and a 400 response. """ if payload.username is None and payload.email is None: - raise ValidationException("At least one of username or email must be provided") + raise HTTPException(status_code=400, detail="Username and email are required") try: if payload.username: existing_user = db.execute(select(User).where(User.username == payload.username)).scalar_one_or_none() if existing_user: - raise ValidationException("Username already exists") + raise HTTPException(status_code=400, detail="Username already exists") user.username = payload.username if payload.email: existing_user = db.execute(select(User).where(User.email == payload.email)).scalar_one_or_none() if existing_user: - raise ValidationException("Email already exists") + raise HTTPException(status_code=400, detail="Email already exists") user.email = payload.email db.commit() db.refresh(user) return user + except HTTPException: + raise except SQLAlchemyError: db.rollback() - raise ValidationException("Database error") + + raise HTTPException(status_code=400, detail="Database error") @router.put("/password") def update_password(payload:UpdatePassword, @@ -539,66 +542,22 @@ def update_password(payload:UpdatePassword, response. """ if not payload.password and not payload.confirm_password: - raise ValidationException("Password and confirm_password are required") + raise HTTPException(status_code=400, detail="Password and confirm_password are required") if len(payload.password) == 0 and len(payload.confirm_password) == 0: - raise ValidationException("Password and confirm_password are required") + raise HTTPException(status_code=400, detail="Password and confirm_password are required") if payload.password != payload.confirm_password: - raise ValidationException("Password and confirm_password are different") + raise HTTPException(status_code=400, detail="Password and confirm_password are different") try: hashed_password = hash_password(payload.password) user.hashed_password = hashed_password db.commit() db.refresh(user) return user + except HTTPException: + raise except SQLAlchemyError: db.rollback() - raise ValidationException("Database error") - - -@router.post("/change-password", response_model=MessageResponse) -def change_password( - payload: ChangePasswordRequest, - user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - """Securely rotate the authenticated user's password. - - Validates that the provided current password matches the stored hash - before updating to the new password. This is the secure, recommended - way for users to change their own password since it confirms the - requester actually knows the existing credentials. - - Args: - payload: ChangePasswordRequest containing `current_password` and `new_password`. - user: The currently authenticated user, obtained from the `get_current_user` dependency. - db: SQLAlchemy database session, obtained from the dependency. - - Returns: - MessageResponse: Confirmation message on success. - - Raises: - HTTPException: 401 if `current_password` does not match the stored hash. - HTTPException: 400 if a database error occurs during commit. - """ - if not verify_password(payload.current_password, user.hashed_password): - raise UnauthorizedException("Current password is incorrect") - - try: - user.hashed_password = hash_password(payload.new_password) - db.commit() - db.refresh(user) - except SQLAlchemyError: - db.rollback() - raise ValidationException("Database error") - - audit_log( - action="user.change_password", - user_id=str(user.id), - result="success", - ) - - return MessageResponse(message="Password updated successfully") - + raise HTTPException(status_code=400, detail="Database error") @router.post("/api-keys", response_model=ApiKeyCreateResponse, status_code=status.HTTP_201_CREATED) def create_api_key( @@ -622,14 +581,6 @@ def create_api_key( db.commit() db.refresh(api_key) - audit_log( - action="api_key.create", - user_id=str(user.id), - result="success", - resource=str(api_key.id), - details={"name": name}, - ) - return ApiKeyCreateResponse( id=str(api_key.id), name=api_key.name, @@ -657,18 +608,10 @@ def delete_api_key(key_id: str, user: User = Depends(get_current_user), db: Sess """Revoke an API key.""" api_key = db.query(ApiKey).filter(ApiKey.id == key_id, ApiKey.user_id == user.id).first() if not api_key: - raise NotFoundException("API key") + raise HTTPException(status_code=404, detail="API key not found") db.delete(api_key) db.commit() - - audit_log( - action="api_key.delete", - user_id=str(user.id), - result="success", - resource=key_id, - ) - return None @router.get("/config") @@ -699,7 +642,10 @@ def _unique_google_username(email: str, db: Session) -> str: def _require_google_drive_config() -> None: if not settings.GOOGLE_CLIENT_ID or not settings.GOOGLE_CLIENT_SECRET: - raise ExternalServiceException("Google Drive", "Google Drive OAuth is not configured") + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Google Drive OAuth is not configured", + ) def _create_google_drive_state(user_id: str) -> str: @@ -871,7 +817,10 @@ def huggingface_login(response: Response): and returns the Hugging Face OAuth authorization URL. """ if not settings.HF_CLIENT_ID or not settings.HF_REDIRECT_URI: - raise ExternalServiceException("Hugging Face", "Hugging Face OAuth is not configured") + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Hugging Face OAuth is not configured", + ) # Generate CSRF state state = secrets.token_urlsafe(32) @@ -915,7 +864,10 @@ async def huggingface_callback( """ # 1. Verify CSRF State if not oauth_state or state != oauth_state: - raise ValidationException("State verification failed. Possible CSRF attack.") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="State verification failed. Possible CSRF attack.", + ) # 2. Exchange code for access_token via Hugging Face API token_url = "https://huggingface.co/oauth/token" @@ -934,13 +886,22 @@ async def huggingface_callback( token_response.raise_for_status() token_data = token_response.json() except httpx.HTTPStatusError as e: - raise UnauthorizedException(f"Failed to exchange code: {e.response.text}") + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=f"Failed to exchange code: {e.response.text}", + ) except Exception as e: - raise AppException("INTERNAL_ERROR", f"Token exchange error: {str(e)}", 500) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Token exchange error: {str(e)}", + ) hf_access_token = token_data.get("access_token") if not hf_access_token: - raise UnauthorizedException("No access token returned from Hugging Face") + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="No access token returned from Hugging Face", + ) # 3. Fetch user profile data via /oauth/userinfo userinfo_url = "https://huggingface.co/oauth/userinfo" @@ -952,13 +913,19 @@ async def huggingface_callback( userinfo_response.raise_for_status() user_data = userinfo_response.json() except Exception as e: - raise AppException("INTERNAL_ERROR", f"Failed to retrieve Hugging Face user info: {str(e)}", 500) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to retrieve Hugging Face user info: {str(e)}", + ) email = user_data.get("email") username = user_data.get("preferred_username") or user_data.get("username") or user_data.get("name") if not email: - raise ValidationException("Hugging Face account email is required but not provided") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Hugging Face account email is required but not provided", + ) email = email.lower() if not username: @@ -987,9 +954,8 @@ async def huggingface_callback( access_token = create_access_token(user.id) refresh_token = create_refresh_token(user.id) - # 6. Set tokens as HttpOnly cookies and Redirect (appending tokens to query parameters for cross-origin frontend support) - frontend_url = settings.FRONTEND_URL.rstrip("/") - redirect_dest = f"{frontend_url}/dashboard?token={access_token}&refresh_token={refresh_token}" + # 6. Set tokens as HttpOnly cookies and Redirect + redirect_dest = f"{settings.FRONTEND_URL}/dashboard" if settings.ENVIRONMENT == "development" else "/dashboard" response = RedirectResponse( url=redirect_dest, status_code=status.HTTP_307_TEMPORARY_REDIRECT, @@ -1026,4 +992,4 @@ def logout(response: Response): """ response.delete_cookie(key="access_token") response.delete_cookie(key="refresh_token") - return {"message": "Successfully logged out"} \ No newline at end of file + return {"message": "Successfully logged out"} diff --git a/backend/app/routes/chat.py b/backend/app/routes/chat.py index 05801fa3..8bfd8c61 100644 --- a/backend/app/routes/chat.py +++ b/backend/app/routes/chat.py @@ -1,7 +1,6 @@ """ Chat routes — ask questions with RAG, stream responses via SSE, manage history. """ - import html import json import time @@ -10,20 +9,15 @@ import logging from typing import Optional, List -from fastapi import APIRouter, Depends, Request, WebSocket, WebSocketDisconnect, Query +from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import Response, StreamingResponse from sqlalchemy.orm import Session from app.auth import get_current_user -from app.cache import get_cached_response, set_cached_response from app.database import get_db -from app.exceptions import ( - NotFoundException, - UnauthorizedException, - ValidationException, ) from app.metrics import record_query_response_time from app.models import User, ChatMessage, Document, SharedMessage, ChatSession -from app.rate_limit import CHAT_QUERY_RATE_LIMIT, check_chat_ws_rate_limit, limiter +from app.rate_limit import CHAT_QUERY_RATE_LIMIT, limiter from app.rag.security import UnsafePromptError, validate_user_input from app.schemas import ( ChatRequest, @@ -35,7 +29,6 @@ ShareLinkResponse, SourceChunk, ChatSessionCreate, - ChatSessionUpdate, ChatSessionResponse, ) @@ -44,181 +37,6 @@ router = APIRouter(prefix="/chat", tags=["Chat"]) -@router.websocket("/ws") -async def chat_ws(websocket: WebSocket, token: Optional[str] = Query(None)): - """WebSocket endpoint for streaming agentic thoughts and tokens. - - Authenticate via `token` query param or expect first JSON message - containing `{token, question, document_id?, session_id?}`. - """ - await websocket.accept() - - # Simple DB-backed auth similar to get_current_user - from app.database import SessionLocal - from app.auth import decode_token - from app.models import ApiKey, User - - db = SessionLocal() - user = None - - try: - # Try token from query param - if token: - tok = token - initial_payload = None - else: - # Expect first message to contain token and the payload - msg = await websocket.receive_json() - tok = msg.get("token") - initial_payload = msg - - if not tok: - await websocket.send_json({"type": "error", "data": "Missing token"}) - await websocket.close() - return - - # API key check - if tok.startswith("pdf_rag_"): - import hashlib - hashed = hashlib.sha256(tok.encode("utf-8")).hexdigest() - api_key = db.query(ApiKey).filter(ApiKey.hashed_key == hashed, ApiKey.is_active == True).first() - if not api_key: - await websocket.send_json({"type": "error", "data": "Invalid API key"}) - await websocket.close() - return - user = api_key.user - else: - user_id = decode_token(tok) - if not user_id: - await websocket.send_json({"type": "error", "data": "Invalid or expired token"}) - await websocket.close() - return - user = db.query(User).filter(User.id == user_id).first() - - if not user: - await websocket.send_json({"type": "error", "data": "User not found"}) - await websocket.close() - return - - # /chat/ask and /chat/ask/stream enforce CHAT_QUERY_RATE_LIMIT via - # @limiter.limit, but that decorator only works on HTTP routes — it - # never runs for this WebSocket handler. Without this check, a client - # could call the same generate_answer_stream(...) RAG/LLM pipeline an - # unbounded number of times per minute by sending requests over /chat/ws - # (or opening multiple connections), completely bypassing the limit. - if not check_chat_ws_rate_limit(user.id): - await websocket.send_json({"type": "error", "data": "Rate limit exceeded"}) - await websocket.close() - return - - # Receive or reuse initial payload - if initial_payload: - payload = initial_payload - else: - payload = await websocket.receive_json() - - question = payload.get("question") - document_id = payload.get("document_id") - session_id = payload.get("session_id") - - from app.rag.security import validate_user_input, UnsafePromptError - - try: - validate_user_input(question) - except UnsafePromptError as exc: - await websocket.send_json({"type": "error", "data": str(exc)}) - await websocket.close() - return - - # Validate document if given - if document_id: - doc = db.query(Document).filter( - Document.id == document_id, - Document.user_id == user.id, - Document.is_deleted.is_(False), - ).first() - if not doc: - await websocket.send_json({"type": "error", "data": "Document not found"}) - await websocket.close() - return - if doc.status != "ready": - progress = getattr(doc, "processing_progress", None) - stage = getattr(doc, "processing_stage", None) - detail = f"Document is still {doc.status}." - if progress is not None: - detail += f" Progress: {progress}%" - if stage: - detail += f" Stage: {stage}" - await websocket.send_json({"type": "error", "data": detail}) - await websocket.close() - return - - # Resolve or create session - if not session_id: - session = db.query(ChatSession).filter(ChatSession.user_id == user.id).first() - if not session: - session = ChatSession(user_id=user.id, title="Default Chat") - db.add(session) - db.commit() - db.refresh(session) - session_id = session.id - - # Build chat history - recent_messages = ( - db.query(ChatMessage) - .filter( - ChatMessage.session_id == session_id, - ChatMessage.user_id == user.id, - ) - .order_by(ChatMessage.created_at.desc()) - .limit(12) - .all() - ) - recent_messages.reverse() - chat_history = [{"role": m.role, "content": m.content} for m in recent_messages] - - # Save user message - _save_message(db, user.id, document_id, "user", question, session_id=session_id) - - # Stream answer using existing generator and forward structured events - try: - for chunk in generate_answer_stream( - question=question, - user_id=user.id, - document_id=document_id, - hf_token=user.hf_token, - chat_history=chat_history, - ): - # chunk is SSE-style string like 'data: {json}\n\n' or similar - try: - if chunk.startswith("data: "): - payload = json.loads(chunk[6:].strip()) - await websocket.send_json(payload) - else: - # Fallback: send raw token - await websocket.send_json({"type": "token", "data": chunk}) - except Exception: - await websocket.send_json({"type": "token", "data": chunk}) - - # Notify client - await websocket.send_json({"type": "done"}) - - except WebSocketDisconnect: - return - except Exception as e: - await websocket.send_json({"type": "error", "data": str(e)}) - - except WebSocketDisconnect: - return - except Exception as e: - try: - await websocket.send_json({"type": "error", "data": str(e)}) - except Exception: - pass - finally: - db.close() - - @router.get( "/share/{message_id}", response_model=ShareAnswerResponse, @@ -232,18 +50,19 @@ def get_shared_answer( message_id: str, db: Session = Depends(get_db), ): - """Return a public shared assistant answer by message ID.""" - message = ( - db.query(ChatMessage) - .filter( - ChatMessage.id == message_id, - ChatMessage.role == "assistant", - ) - .first() - ) + """Return a public shared assistant answer by message ID. + + Only assistant messages that already have a `SharedMessage` record are + exposed. User prompts, private chat history, and unshared answers remain + protected. + """ + message = db.query(ChatMessage).filter( + ChatMessage.id == message_id, + ChatMessage.role == "assistant", + ).first() if not message or not db.query(SharedMessage).filter(SharedMessage.message_id == message.id).first(): - raise NotFoundException("Shared answer") + raise HTTPException(status_code=404, detail="Shared answer not found") return _share_answer_response(message) @@ -253,7 +72,8 @@ def get_shared_answer( response_model=ShareLinkResponse, summary="Create a public share link for an assistant answer", description=( - "Marks one authenticated user's assistant message as shareable and " "returns the frontend share URL." + "Marks one authenticated user's assistant message as shareable and " + "returns the frontend share URL." ), ) def create_share_link( @@ -261,21 +81,21 @@ def create_share_link( user: User = Depends(get_current_user), db: Session = Depends(get_db), ): - """Create or reuse a public share record for an assistant answer.""" - message = ( - db.query(ChatMessage) - .filter( - ChatMessage.id == message_id, - ChatMessage.user_id == user.id, - ) - .first() - ) + """Create or reuse a public share record for an assistant answer. + + The message must belong to the authenticated user and must have the + assistant role. User-authored messages cannot be shared through this route. + """ + message = db.query(ChatMessage).filter( + ChatMessage.id == message_id, + ChatMessage.user_id == user.id, + ).first() if not message: - raise NotFoundException("Message") + raise HTTPException(status_code=404, detail="Message not found") if message.role != "assistant": - raise ValidationException("Only assistant messages can be shared") + raise HTTPException(status_code=400, detail="Only assistant messages can be shared") shared_message = db.query(SharedMessage).filter(SharedMessage.message_id == message.id).first() if not shared_message: @@ -301,7 +121,10 @@ def get_chat_sessions( ): """Retrieve all chat sessions for the authenticated user.""" sessions = ( - db.query(ChatSession).filter(ChatSession.user_id == user.id).order_by(ChatSession.created_at.desc()).all() + db.query(ChatSession) + .filter(ChatSession.user_id == user.id) + .order_by(ChatSession.created_at.desc()) + .all() ) return sessions @@ -351,36 +174,7 @@ def rename_chat_session( .first() ) if not session: - raise NotFoundException("Chat session") - session.title = payload.title - db.commit() - db.refresh(session) - return session - - -@router.patch( - "/sessions/{session_id}", - response_model=ChatSessionResponse, - summary="Update a chat session title", - description="Partially updates a chat session title after verifying ownership.", -) -def update_chat_session( - session_id: str, - payload: ChatSessionUpdate, - user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - """Update the title of an existing chat session owned by the authenticated user.""" - session = ( - db.query(ChatSession) - .filter( - ChatSession.id == session_id, - ChatSession.user_id == user.id, - ) - .first() - ) - if not session: - raise NotFoundException("Chat session") + raise HTTPException(status_code=404, detail="Chat session not found") session.title = payload.title db.commit() db.refresh(session) @@ -407,7 +201,7 @@ def delete_chat_session( .first() ) if not session: - raise NotFoundException("Chat session") + raise HTTPException(status_code=404, detail="Chat session not found") db.delete(session) db.commit() return Response(status_code=204) @@ -434,7 +228,7 @@ def get_session_history( .first() ) if not session: - raise NotFoundException("Chat session") + raise HTTPException(status_code=404, detail="Chat session not found") messages = ( db.query(ChatMessage) @@ -468,48 +262,16 @@ def get_session_history( return ChatHistoryResponse(messages=formatted, document_id=None) -def generate_answer( - question: str, - user_id: str, - document_id: Optional[str] = None, - document_ids: Optional[List[str]] = None, - hf_token: Optional[str] = None, - top_k: Optional[int] = None, - chat_history: Optional[list] = None, -): +def generate_answer(question: str, user_id: str, document_id: Optional[str] = None, hf_token: Optional[str] = None, top_k: Optional[int] = None, chat_history: Optional[list] = None): from app.rag.agent import generate_answer as _generate_answer - return _generate_answer( - question=question, - user_id=user_id, - document_id=document_id, - document_ids=document_ids, - hf_token=hf_token, - top_k=top_k, - chat_history=chat_history, - ) + return _generate_answer(question=question, user_id=user_id, document_id=document_id, hf_token=hf_token, top_k=top_k, chat_history=chat_history) -def generate_answer_stream( - question: str, - user_id: str, - document_id: Optional[str] = None, - document_ids: Optional[List[str]] = None, - hf_token: Optional[str] = None, - top_k: Optional[int] = None, - chat_history: Optional[list] = None, -): +def generate_answer_stream(question: str, user_id: str, document_id: Optional[str] = None, hf_token: Optional[str] = None, top_k: Optional[int] = None, chat_history: Optional[list] = None): from app.rag.agent import generate_answer_stream as _generate_answer_stream - return _generate_answer_stream( - question=question, - user_id=user_id, - document_id=document_id, - document_ids=document_ids, - hf_token=hf_token, - top_k=top_k, - chat_history=chat_history, - ) + return _generate_answer_stream(question=question, user_id=user_id, document_id=document_id, hf_token=hf_token, top_k=top_k, chat_history=chat_history) @router.post( @@ -523,83 +285,40 @@ def generate_answer_stream( ) @limiter.limit(CHAT_QUERY_RATE_LIMIT) def ask_question( + request: Request, payload: ChatRequest, - request: Request = None, user: User = Depends(get_current_user), db: Session = Depends(get_db), ): """Ask a question with RAG retrieval and return the complete answer.""" started_at = time.perf_counter() - - # Bind query parameters to request context variables and state - if request is not None: - request.state.query = payload.question - from app.observability import query_text_var - query_text_var.set(payload.question) - logger.info(f"Processing RAG chat query: '{payload.question}'") - try: try: validate_user_input(payload.question) except UnsafePromptError as exc: - raise ValidationException(str(exc)) from exc + raise HTTPException(status_code=400, detail=str(exc)) from exc # Validate document exists if specified if payload.document_id: - doc = ( - db.query(Document) - .filter( - Document.id == payload.document_id, - Document.user_id == user.id, - Document.is_deleted.is_(False), - ) - .first() - ) + doc = db.query(Document).filter( + Document.id == payload.document_id, + Document.user_id == user.id, + Document.is_deleted.is_(False), + ).first() if not doc: - raise NotFoundException("Document") + raise HTTPException(status_code=404, detail="Document not found") if doc.status != "ready": - progress = getattr(doc, "processing_progress", None) - stage = getattr(doc, "processing_stage", None) - detail = f"Document is still {doc.status}. Please wait for processing to complete." - if progress is not None: - detail += f" Progress: {progress}%" - if stage: - detail += f" Stage: {stage}" - raise ValidationException(detail) + raise HTTPException( + status_code=400, + detail=f"Document is still {doc.status}. Please wait for processing to complete.", + ) # Update last_accessed_at timestamp doc.last_accessed_at = datetime.now(timezone.utc) db.commit() - # Validate documents if multiple specified - elif payload.document_ids: - docs = ( - db.query(Document) - .filter( - Document.id.in_(payload.document_ids), - Document.user_id == user.id, - Document.is_deleted.is_(False), - ) - .all() - ) - - found_ids = {doc.id for doc in docs} - missing = [doc_id for doc_id in payload.document_ids if doc_id not in found_ids] - if missing: - raise NotFoundException("Document") - - not_ready = [doc.original_name for doc in docs if doc.status != "ready"] - if not_ready: - raise ValidationException( - f"Some documents are still processing: {', '.join(not_ready)}. Please wait." - ) - - for doc in docs: - doc.last_accessed_at = datetime.now(timezone.utc) - db.commit() - # Resolve or create session session_id = payload.session_id if not session_id: @@ -625,54 +344,18 @@ def ask_question( recent_messages.reverse() chat_history = [{"role": m.role, "content": m.content} for m in recent_messages] - cache_doc_key = str(payload.document_id or "") - if payload.document_ids: - cache_doc_key = "multi:" + ",".join(sorted(payload.document_ids)) - # Cache check — return instantly if this (user, document, question) was answered before - cached_answer = get_cached_response( - user_id=user.id, - document_id=cache_doc_key, - question=payload.question, - ) - if cached_answer is not None: - logger.debug("Returning cached response for question: %s", payload.question[:40]) - return ChatResponse( - answer=cached_answer, - sources=[], - document_id=payload.document_id, - ) - result = generate_answer( question=payload.question, user_id=user.id, document_id=payload.document_id, - document_ids=payload.document_ids, hf_token=user.hf_token, top_k=payload.top_k, chat_history=chat_history, ) - # Bind chunks retrieved to request state and context variables - chunks_count = len(result.get("sources", [])) - if request is not None: - request.state.chunks_retrieved = chunks_count - from app.observability import chunks_retrieved_var - chunks_retrieved_var.set(chunks_count) - logger.info(f"RAG chat query processed successfully, retrieved {chunks_count} chunks") - - # Store result in cache for future identical questions - set_cached_response( - user_id=user.id, - document_id=cache_doc_key, - question=payload.question, - answer=result["answer"], - ) - # Save to chat history - _save_message(db, user.id, cache_doc_key, "user", payload.question, session_id=session_id) - _save_message( - db, user.id, cache_doc_key, "assistant", result["answer"], result["sources"], session_id=session_id - ) + _save_message(db, user.id, payload.document_id, "user", payload.question, session_id=session_id) + _save_message(db, user.id, payload.document_id, "assistant", result["answer"], result["sources"], session_id=session_id) return ChatResponse( answer=result["answer"], @@ -693,80 +376,38 @@ def ask_question( ) @limiter.limit(CHAT_QUERY_RATE_LIMIT) def ask_question_stream( + request: Request, payload: ChatRequest, - request: Request = None, user: User = Depends(get_current_user), db: Session = Depends(get_db), ): """Ask a question and stream the answer using Server-Sent Events.""" - # Bind query parameters to request context variables and state - if request is not None: - request.state.query = payload.question - from app.observability import query_text_var - query_text_var.set(payload.question) - logger.info(f"Processing streaming RAG chat query: '{payload.question}'") - try: validate_user_input(payload.question) except UnsafePromptError as exc: - raise ValidationException(str(exc)) from exc + raise HTTPException(status_code=400, detail=str(exc)) from exc # Validate document if payload.document_id: - doc = ( - db.query(Document) - .filter( - Document.id == payload.document_id, - Document.user_id == user.id, - Document.is_deleted.is_(False), - ) - .first() - ) + doc = db.query(Document).filter( + Document.id == payload.document_id, + Document.user_id == user.id, + Document.is_deleted.is_(False), + ).first() if not doc: - raise NotFoundException("Document") + raise HTTPException(status_code=404, detail="Document not found") if doc.status != "ready": - progress = getattr(doc, "processing_progress", None) - stage = getattr(doc, "processing_stage", None) - detail = f"Document is still {doc.status}. Please wait for processing to complete." - if progress is not None: - detail += f" Progress: {progress}%" - if stage: - detail += f" Stage: {stage}" - raise ValidationException(detail) + raise HTTPException( + status_code=400, + detail=f"Document is still {doc.status}. Please wait for processing to complete.", + ) # Update last_accessed_at timestamp doc.last_accessed_at = datetime.now(timezone.utc) db.commit() - # Validate documents if multiple specified - elif payload.document_ids: - docs = ( - db.query(Document) - .filter( - Document.id.in_(payload.document_ids), - Document.user_id == user.id, - Document.is_deleted.is_(False), - ) - .all() - ) - - found_ids = {doc.id for doc in docs} - missing = [doc_id for doc_id in payload.document_ids if doc_id not in found_ids] - if missing: - raise NotFoundException("Document") - - not_ready = [doc.original_name for doc in docs if doc.status != "ready"] - if not_ready: - raise ValidationException( - f"Some documents are still processing: {', '.join(not_ready)}. Please wait." - ) - - for doc in docs: - doc.last_accessed_at = datetime.now(timezone.utc) - db.commit() - started_at = time.perf_counter() # Resolve or create session @@ -794,37 +435,8 @@ def ask_question_stream( recent_messages.reverse() chat_history = [{"role": m.role, "content": m.content} for m in recent_messages] - cache_doc_key = str(payload.document_id or "") - if payload.document_ids: - cache_doc_key = "multi:" + ",".join(sorted(payload.document_ids)) - # Save user message immediately - _save_message(db, user.id, cache_doc_key, "user", payload.question, session_id=session_id) - - # Cache check before starting the stream - cached_answer = get_cached_response( - user_id=user.id, - document_id=cache_doc_key, - question=payload.question, - ) - if cached_answer is not None: - logger.debug("Returning cached stream response for question: %s", payload.question[:40]) - - async def cached_event_stream(): - payload_json = json.dumps({"type": "token", "data": cached_answer}) - yield f"data: {payload_json}\n\n" - done_json = json.dumps({"type": "done"}) - yield f"data: {done_json}\n\n" - - return StreamingResponse( - cached_event_stream(), - media_type="text/event-stream", - headers={ - "Cache-Control": "no-cache", - "Connection": "keep-alive", - "X-Accel-Buffering": "no", - }, - ) + _save_message(db, user.id, payload.document_id, "user", payload.question, session_id=session_id) # Stream response def event_stream(): @@ -835,53 +447,31 @@ def event_stream(): for chunk in generate_answer_stream( question=payload.question, user_id=user.id, - document_id=cache_doc_key, - document_ids=payload.document_ids, + document_id=payload.document_id, hf_token=user.hf_token, top_k=payload.top_k, chat_history=chat_history, ): + yield chunk + # Parse to accumulate full answer for history try: if chunk.startswith("data: "): data = json.loads(chunk[6:].strip()) - if data.get("type") == "done": - continue # We will yield our own done event with response time if data.get("type") == "token": full_answer += data.get("data", "") elif data.get("type") == "sources": sources = data.get("data", []) except Exception: pass - yield chunk - - # Cache the full answer for future identical questions - if full_answer: - set_cached_response( - user_id=user.id, - document_id=cache_doc_key, - question=payload.question, - answer=full_answer, - ) # Save assistant response to history - from app.database import get_db_session - - with get_db_session() as save_db: - _save_message( - save_db, user.id, cache_doc_key, "assistant", full_answer, sources, session_id=session_id - ) - - # Log streaming response RAG completion - chunks_count = len(sources) - from app.observability import chunks_retrieved_var, query_text_var, user_id_var - user_id_var.set(user.id) - query_text_var.set(payload.question) - chunks_retrieved_var.set(chunks_count) - logger.info(f"Streaming RAG chat query completed, retrieved {chunks_count} chunks") - - elapsed_ms = round((time.perf_counter() - started_at) * 1000) - yield f"data: {json.dumps({'type': 'done', 'response_time_ms': elapsed_ms})}\n\n" + from app.database import SessionLocal + save_db = SessionLocal() + try: + _save_message(save_db, user.id, payload.document_id, "assistant", full_answer, sources, session_id=session_id) + finally: + save_db.close() finally: record_query_response_time(time.perf_counter() - started_at) @@ -927,16 +517,14 @@ def get_chat_history( except Exception: pass - formatted.append( - ChatMessageResponse( - id=str(msg.id), - role=msg.role, - content=msg.content, - sources=sources, - feedback=msg.feedback, - created_at=msg.created_at, - ) - ) + formatted.append(ChatMessageResponse( + id=str(msg.id), + role=msg.role, + content=msg.content, + sources=sources, + feedback=msg.feedback, + created_at=msg.created_at, + )) return ChatHistoryResponse(messages=formatted, document_id=document_id) @@ -965,12 +553,11 @@ def export_chat_history( resolved_user = db.query(User).filter(User.id == user_id).first() if resolved_user is None: - raise UnauthorizedException("Authentication required") + raise HTTPException(status_code=401, detail="Authentication required") if format not in ("md", "txt", "pdf"): - raise ValidationException("Format must be 'md', 'txt', or 'pdf'") + raise HTTPException(status_code=400, detail="Format must be 'md', 'txt', or 'pdf'") - # Verify document exists and belongs to user doc = db.query(Document).filter( Document.id == document_id, Document.user_id == resolved_user.id, @@ -978,7 +565,7 @@ def export_chat_history( ).first() if not doc: - raise NotFoundException("Document") + raise HTTPException(status_code=404, detail="Document not found") messages = ( db.query(ChatMessage) @@ -991,7 +578,7 @@ def export_chat_history( ) if not messages: - raise NotFoundException("Chat history") + raise HTTPException(status_code=404, detail="No chat history found for this document") if format == "md": content = _format_markdown(doc, messages) @@ -1003,7 +590,6 @@ def export_chat_history( extension = "txt" else: from app.routes.chat_export import format_pdf as _format_pdf - content = _format_pdf(doc, messages) media_type = "application/pdf" extension = "pdf" @@ -1031,22 +617,10 @@ def clear_chat_history( db: Session = Depends(get_db), ): """Delete all chat messages associated with a specific document.""" - # Find the query/subquery of chat messages to delete - message_ids_query = db.query(ChatMessage.id).filter( - ChatMessage.user_id == user.id, - ChatMessage.document_id == document_id, - ) - - # Delete any associated SharedMessage records first - db.query(SharedMessage).filter( - SharedMessage.message_id.in_(message_ids_query) - ).delete(synchronize_session=False) - - # Delete the ChatMessage records db.query(ChatMessage).filter( ChatMessage.user_id == user.id, ChatMessage.document_id == document_id, - ).delete(synchronize_session=False) + ).delete() db.commit() return {"message": "Chat history cleared"} @@ -1066,9 +640,9 @@ def submit_feedback( ).first() if not msg: - raise NotFoundException("Message") + raise HTTPException(status_code=404, detail="Message not found") if msg.role != "assistant": - raise ValidationException("Can only provide feedback on assistant messages") + raise HTTPException(status_code=400, detail="Can only provide feedback on assistant messages") msg.feedback = payload.feedback db.commit() @@ -1161,11 +735,9 @@ def _format_markdown(doc, messages) -> str: lines.append("**Sources:**") lines.append("") for i, src in enumerate(sources, 1): - lines.append( - f"> **[{i}]** {src.get('filename', 'Unknown')}, " - f"Page {src.get('page', '?')} " - f"(Confidence: {src.get('confidence', 0)}%)" - ) + lines.append(f"> **[{i}]** {src.get('filename', 'Unknown')}, " + f"Page {src.get('page', '?')} " + f"(Confidence: {src.get('confidence', 0)}%)") text_preview = src.get("text", "")[:150] if text_preview: lines.append(f"> {text_preview}...") @@ -1204,15 +776,13 @@ def _format_plaintext(doc, messages) -> str: lines.append("") lines.append("Sources:") for i, src in enumerate(sources, 1): - lines.append( - f" [{i}] {src.get('filename', 'Unknown')}, " - f"Page {src.get('page', '?')} " - f"(Confidence: {src.get('confidence', 0)}%)" - ) + lines.append(f" [{i}] {src.get('filename', 'Unknown')}, " + f"Page {src.get('page', '?')} " + f"(Confidence: {src.get('confidence', 0)}%)") except Exception: pass lines.append("-" * 60) lines.append("") - return "\n".join(lines) \ No newline at end of file + return "\n".join(lines) diff --git a/backend/app/routes/chat_export.py b/backend/app/routes/chat_export.py deleted file mode 100644 index 0dfffb2e..00000000 --- a/backend/app/routes/chat_export.py +++ /dev/null @@ -1,269 +0,0 @@ -""" -PDF export helper for chat transcripts. - -Called by the /chat/export/{document_id}?format=pdf route in chat.py. -Uses ReportLab (already in requirements.txt) to produce a clean, readable -PDF from a list of ChatMessage ORM objects. -""" -import json -import textwrap -from datetime import datetime -from io import BytesIO -from typing import List - -from reportlab.lib import colors -from reportlab.lib.enums import TA_LEFT, TA_RIGHT -from reportlab.lib.pagesizes import A4 -from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet -from reportlab.lib.units import mm -from reportlab.platypus import ( - HRFlowable, - Paragraph, - SimpleDocTemplate, - Spacer, - Table, - TableStyle, -) - -# ── Palette ─────────────────────────────────────────────────────────────────── -_PRIMARY = colors.HexColor("#4F46E5") # indigo — user bubble accent -_ASSISTANT = colors.HexColor("#059669") # emerald — assistant bubble accent -_MUTED = colors.HexColor("#6B7280") # gray-500 — timestamps / meta -_SOURCE_BG = colors.HexColor("#F3F4F6") # gray-100 — source card background -_DIVIDER = colors.HexColor("#E5E7EB") # gray-200 — horizontal rules -_BLACK = colors.HexColor("#111827") # near-black — body text - - -def _build_styles() -> dict: - """Return a dict of named ParagraphStyles used throughout the PDF.""" - base = getSampleStyleSheet() - - return { - "title": ParagraphStyle( - "Title", - parent=base["Normal"], - fontSize=20, - leading=26, - textColor=_BLACK, - spaceAfter=2 * mm, - fontName="Helvetica-Bold", - ), - "meta": ParagraphStyle( - "Meta", - parent=base["Normal"], - fontSize=9, - leading=13, - textColor=_MUTED, - spaceAfter=4 * mm, - fontName="Helvetica", - ), - "role_user": ParagraphStyle( - "RoleUser", - parent=base["Normal"], - fontSize=10, - leading=14, - textColor=_PRIMARY, - fontName="Helvetica-Bold", - spaceAfter=1 * mm, - ), - "role_assistant": ParagraphStyle( - "RoleAssistant", - parent=base["Normal"], - fontSize=10, - leading=14, - textColor=_ASSISTANT, - fontName="Helvetica-Bold", - spaceAfter=1 * mm, - ), - "timestamp": ParagraphStyle( - "Timestamp", - parent=base["Normal"], - fontSize=8, - leading=11, - textColor=_MUTED, - fontName="Helvetica-Oblique", - spaceAfter=2 * mm, - ), - "body": ParagraphStyle( - "Body", - parent=base["Normal"], - fontSize=10, - leading=15, - textColor=_BLACK, - fontName="Helvetica", - spaceAfter=3 * mm, - wordWrap="LTR", - ), - "source_label": ParagraphStyle( - "SourceLabel", - parent=base["Normal"], - fontSize=8, - leading=11, - textColor=_MUTED, - fontName="Helvetica-Bold", - spaceAfter=1 * mm, - ), - "source_item": ParagraphStyle( - "SourceItem", - parent=base["Normal"], - fontSize=8, - leading=12, - textColor=_BLACK, - fontName="Helvetica", - leftIndent=4 * mm, - ), - "source_preview": ParagraphStyle( - "SourcePreview", - parent=base["Normal"], - fontSize=8, - leading=12, - textColor=_MUTED, - fontName="Helvetica-Oblique", - leftIndent=6 * mm, - spaceAfter=1 * mm, - ), - } - - -def _safe_text(text: str) -> str: - """Escape XML special characters so ReportLab Paragraph doesn't crash.""" - return ( - text.replace("&", "&") - .replace("<", "<") - .replace(">", ">") - .replace('"', """) - ) - - -def _wrap_body(text: str, width: int = 100) -> str: - """Soft-wrap long lines so they fit inside the PDF column.""" - paragraphs = text.split("\n") - wrapped = [] - for para in paragraphs: - if len(para) <= width: - wrapped.append(_safe_text(para)) - else: - wrapped.extend(_safe_text(line) for line in textwrap.wrap(para, width)) - return "
".join(wrapped) if wrapped else " " - - -def format_pdf(doc, messages: List) -> bytes: - """Render chat history as a PDF and return the raw bytes. - - Args: - doc: SQLAlchemy Document ORM object (needs .original_name). - messages: Ordered list of ChatMessage ORM objects. - - Returns: - Raw PDF bytes ready to be returned as a FastAPI Response. - """ - buffer = BytesIO() - styles = _build_styles() - - page_w, page_h = A4 - margin = 18 * mm - - pdf = SimpleDocTemplate( - buffer, - pagesize=A4, - leftMargin=margin, - rightMargin=margin, - topMargin=20 * mm, - bottomMargin=20 * mm, - title=f"Chat History — {doc.original_name}", - author="PDF Assistant RAG", - ) - - story = [] - - # ── Cover block ─────────────────────────────────────────────────────────── - story.append(Paragraph("Chat Transcript", styles["title"])) - story.append( - Paragraph( - f"Document: {_safe_text(doc.original_name)}
" - f"Exported: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
" - f"Messages: {len(messages)}", - styles["meta"], - ) - ) - story.append( - HRFlowable( - width="100%", - thickness=1, - color=_PRIMARY, - spaceAfter=6 * mm, - ) - ) - - # ── Messages ────────────────────────────────────────────────────────────── - for msg in messages: - is_user = msg.role == "user" - role_label = "You" if is_user else "Assistant" - role_style = styles["role_user"] if is_user else styles["role_assistant"] - - timestamp = ( - msg.created_at.strftime("%Y-%m-%d %H:%M:%S") - if msg.created_at - else "" - ) - - # Role + timestamp - story.append(Paragraph(role_label, role_style)) - if timestamp: - story.append(Paragraph(timestamp, styles["timestamp"])) - - # Message body — handle multi-line content - body_html = _wrap_body(msg.content or "") - story.append(Paragraph(body_html, styles["body"])) - - # Sources (assistant messages only) - if not is_user and msg.sources_json: - try: - sources = json.loads(msg.sources_json) - if sources: - story.append(Paragraph("Sources:", styles["source_label"])) - for i, src in enumerate(sources, 1): - filename = _safe_text(src.get("filename", "Unknown")) - page = src.get("page", "?") - confidence = src.get("confidence", 0) - preview = _safe_text(src.get("text", "")[:120]) - - story.append( - Paragraph( - f"[{i}] {filename} — Page {page} " - f"(Confidence: {confidence}%)", - styles["source_item"], - ) - ) - if preview: - story.append( - Paragraph( - f"{preview}{'...' if len(src.get('text','')) > 120 else ''}", - styles["source_preview"], - ) - ) - except Exception: - pass - - # Divider between messages - story.append( - HRFlowable( - width="100%", - thickness=0.5, - color=_DIVIDER, - spaceBefore=2 * mm, - spaceAfter=4 * mm, - ) - ) - - # ── Footer note ─────────────────────────────────────────────────────────── - story.append(Spacer(1, 4 * mm)) - story.append( - Paragraph( - f"Generated by PDF Assistant RAG · {datetime.now().strftime('%Y-%m-%d')}", - styles["meta"], - ) - ) - - pdf.build(story) - return buffer.getvalue() diff --git a/backend/app/routes/documents.py b/backend/app/routes/documents.py index de131b36..73ce58ef 100644 --- a/backend/app/routes/documents.py +++ b/backend/app/routes/documents.py @@ -9,42 +9,29 @@ import asyncio import concurrent.futures from datetime import datetime, timezone -from typing import List, Optional +from typing import Optional from pathlib import Path import shutil -import socket -import ipaddress import tempfile from urllib.parse import urlparse -from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, status, Query, BackgroundTasks, Request, Form +from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, status, Query, BackgroundTasks from fastapi.responses import FileResponse from sqlalchemy.orm import Session -from sqlalchemy import select, func from app.database import get_db -from app.exceptions import ( - ExternalServiceException, - NotFoundException, - ValidationException, - ConflictException, - AppException, - ForbiddenException, -) from app.models import User, Document from app.schemas import ( DocumentResponse, DocumentListResponse, DocumentStatusResponse, - DocumentUpdate, + DocumentRename, ChunkSettings, UploadUrl, - BatchUploadResponse, ) from app.auth import get_current_user from app.config import get_settings from app.tasks import process_document from app.services.document_ingestion import ingest_document -from app.services.layout_parser import AdvancedPDFParser try: from crawl4ai import AsyncWebCrawler @@ -66,54 +53,6 @@ ALLOWED_MIME_TYPES = settings.ALLOWED_MIME_TYPES -def _deserialize_doc(doc: Document) -> DocumentResponse: - """Return a DocumentResponse with extracted_urls parsed from JSON string.""" - import json as _json - response = DocumentResponse.model_validate(doc) - if doc.extracted_urls: - try: - response = response.model_copy( - update={"extracted_urls": _json.loads(doc.extracted_urls)} - ) - except Exception: - response = response.model_copy(update={"extracted_urls": []}) - return response - -def _get_documents_query( - db: Session, - user_id: str, - q: Optional[str] = None, -): - """ - Build a filtered SQLAlchemy select query for documents belonging to a user. - - Applies an optional case-insensitive substring filter on ``original_name``. - Does NOT apply pagination – callers are responsible for ``.limit()`` / - ``.offset()`` so this helper stays reusable. - - Args: - db: Active database session. - user_id: ID of the authenticated user whose documents to query. - q: Optional keyword to filter document names (case-insensitive). - - Returns: - A SQLAlchemy ``Select`` statement ready for count or paginated execution. - """ - base_query = ( - select(Document) - .where( - Document.user_id == user_id, - Document.is_deleted.is_(False), - ) - ) - - if q and q.strip(): - pattern = f"%{q.strip()}%" - base_query = base_query.where( - Document.original_name.ilike(pattern) - ) - - return base_query async def validate_upload(file: UploadFile): """Validate an uploaded file and save it to a temporary file. @@ -140,13 +79,13 @@ async def validate_upload(file: UploadFile): - 'python-magic' dependency is missing on the server. """ if not file.filename: - raise ValidationException("No filename provided") + raise HTTPException(status_code=400, detail="No filename provided") ext = Path(file.filename).suffix.lower() # extension without leading dot in settings if ext.lstrip(".") not in settings.ALLOWED_EXTENSIONS: - raise ValidationException("Only PDF, DOCX, TEXT, AND MARKDOWN files are allowed") + raise HTTPException(status_code=400, detail="Only PDF, DOCX, TEXT, AND MARKDOWN files are allowed") # save to a temporary file with tempfile.NamedTemporaryFile(delete=False, suffix=ext) as tmp: @@ -159,7 +98,7 @@ async def validate_upload(file: UploadFile): if size > settings.MAX_UPLOAD_SIZE_MB * 1024 * 1024: Path(temp_path).unlink(missing_ok=True) - raise ValidationException("File too large") + raise HTTPException(status_code=400, detail="File too large") # libmagic may not be installed in all environments — import lazily try: @@ -167,13 +106,13 @@ async def validate_upload(file: UploadFile): # make sure you have installed libmagic in your system, otherwise it will not work except Exception: Path(temp_path).unlink(missing_ok=True) - raise ExternalServiceException("dependency", "Server missing 'python-magic' dependency") + raise HTTPException(status_code=500, detail="Server missing 'python-magic' dependency") mime = magic.from_file(temp_path, mime=True) if mime not in ALLOWED_MIME_TYPES.get(ext, []): Path(temp_path).unlink(missing_ok=True) - raise ValidationException(f"Invalid file type: {mime}") + raise HTTPException(status_code=400, detail=f"Invalid file type: {mime}") # Deep validation: try to parse the file — import parsers lazily try: @@ -187,7 +126,7 @@ async def validate_upload(file: UploadFile): DocxDocument(temp_path) except Exception: Path(temp_path).unlink(missing_ok=True) - raise ValidationException("Corrupted or invalid file") + raise HTTPException(status_code=400, detail="Corrupted or invalid file") return temp_path @@ -229,28 +168,47 @@ async def _crawl(): @router.post("/upload", response_model=DocumentResponse, status_code=status.HTTP_202_ACCEPTED) async def upload_document( - request: Request = None, file: UploadFile = File(...), - chunk_size: int = Form(1000), - chunk_overlap: int = Form(200), background_tasks: BackgroundTasks = None, user: User = Depends(get_current_user), db: Session = Depends(get_db), ): + """ + Upload a document and enqueue RAG processing. + + Validates the uploaded file (extension, size, MIME type, integrity), + saves it to the user's directory, creates a database record with status + 'pending', queues a Celery task for chunking and embedding, and returns + 202 Accepted immediately so large documents do not block the API request + while embeddings are generated. + + Args: + file: The uploaded file, provided as a multipart/form-data field in the request. + background_tasks: FastAPI BackgroundTasks instance for in-process fallback execution. + user: The currently authenticated user, injected by the `get_current_user` dependency. + db: Database session, injected by the `get_db` dependency. + + Returns: + DocumentResponse: The created document record, validated against the + response model (includes id, filename, original_name, file_size, status, etc.). + + Raises: + HTTPException: With status code 400 if: + - No filename is provided. + - The file extension is not allowed. (only .pdf or .docx) + - The file fails validation checks (size, MIME type, integrity). + HTTPException: With status code 500 if: + - The server lacks the 'python-magic' dependency. + """ # ── Validate file type ─────────────────────────── if not file.filename: - raise ValidationException("No filename provided") - - # ── Validate chunking params ───────────────────── - if chunk_size < 100 or chunk_size > 2000: - raise ValidationException("Chunk size must be between 100 and 2000") - if chunk_overlap < 0 or chunk_overlap >= chunk_size: - raise ValidationException("Chunk overlap must be non-negative and less than chunk_size") + raise HTTPException(status_code=400, detail="No filename provided") ext = file.filename.rsplit(".", 1)[-1].lower() if ext not in settings.ALLOWED_EXTENSIONS: - raise ValidationException( - f"File type '.{ext}' not supported. Allowed: {', '.join(settings.ALLOWED_EXTENSIONS)}", + raise HTTPException( + status_code=400, + detail=f"File type '.{ext}' not supported. Allowed: {', '.join(settings.ALLOWED_EXTENSIONS)}", ) # ── Validate and save file to disk ─────────────── @@ -262,7 +220,9 @@ async def upload_document( stored_filename = f"{uuid.uuid4().hex}.{ext}" filepath = os.path.join(user_dir, stored_filename) + # Move temp file to final destination shutil.move(temp_path, filepath) + file_size = Path(filepath).stat().st_size # ── Create database record ─────────────────────── @@ -272,8 +232,6 @@ async def upload_document( original_name=file.filename, file_size=file_size, status="pending", - chunk_size=chunk_size, - chunk_overlap=chunk_overlap ) db.add(document) db.commit() @@ -303,128 +261,9 @@ async def upload_document( return DocumentResponse.model_validate(document).model_copy(update={"task_id": task_id}) -@router.post("/upload/batch", response_model=BatchUploadResponse, status_code=status.HTTP_202_ACCEPTED) -async def batch_upload_documents( - files: List[UploadFile] = File(...), - chunk_size: int = Form(1000), - chunk_overlap: int = Form(200), - background_tasks: BackgroundTasks = None, - user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - """Accept multiple files and enqueue parallel ingestion tasks. - - Each file is validated and saved independently. Successfully saved files - are committed to the database and dispatched to Celery (or the in-process - fallback). Files that fail validation are recorded in the ``failed`` list - and do not block the remaining uploads. - - Args: - files: One or more uploaded files (PDF, DOCX, TXT, MD). - chunk_size: Text chunk size for RAG ingestion (100–2000). - chunk_overlap: Overlap between consecutive chunks (0 < chunk_size). - background_tasks: FastAPI hook for in-process fallback execution. - user: Authenticated user injected by get_current_user. - db: Database session injected by get_db. - - Returns: - BatchUploadResponse with the list of created documents, their task - IDs, total accepted count, and any filenames that were rejected. - """ - if not files: - raise ValidationException("No files provided") - - if chunk_size < 100 or chunk_size > 2000: - raise ValidationException("Chunk size must be between 100 and 2000") - if chunk_overlap < 0 or chunk_overlap >= chunk_size: - raise ValidationException("Chunk overlap must be non-negative and less than chunk_size") - - user_dir = os.path.join(settings.UPLOAD_DIR, user.id) - os.makedirs(user_dir, exist_ok=True) - - created_documents: List[DocumentResponse] = [] - task_ids: List[str] = [] - failed: List[str] = [] - - for file in files: - filename = file.filename or "unknown" - try: - if not file.filename: - raise ValidationException("No filename provided") - - ext = file.filename.rsplit(".", 1)[-1].lower() - if ext not in settings.ALLOWED_EXTENSIONS: - raise ValidationException( - f"File type '.{ext}' not supported. Allowed: {', '.join(settings.ALLOWED_EXTENSIONS)}" - ) - - temp_path = await validate_upload(file) - - stored_filename = f"{uuid.uuid4().hex}.{ext}" - filepath = os.path.join(user_dir, stored_filename) - shutil.move(temp_path, filepath) - file_size = Path(filepath).stat().st_size - - document = Document( - user_id=user.id, - filename=stored_filename, - original_name=file.filename, - file_size=file_size, - status="pending", - chunk_size=chunk_size, - chunk_overlap=chunk_overlap, - ) - db.add(document) - db.commit() - db.refresh(document) - - task_id = None - try: - task = process_document.delay( - document_id=document.id, - filepath=filepath, - original_name=file.filename, - user_id=user.id, - ) - task_id = task.id - except Exception as e: - logger.warning(f"Celery queue failed for {file.filename}, falling back to background task: {e}") - if background_tasks: - background_tasks.add_task( - ingest_document, - document_id=document.id, - filepath=filepath, - original_name=file.filename, - user_id=user.id, - ) - task_id = f"local_{uuid.uuid4().hex}" - - created_documents.append( - DocumentResponse.model_validate(document).model_copy(update={"task_id": task_id}) - ) - task_ids.append(task_id) - - except (ValidationException, ExternalServiceException) as e: - logger.warning(f"Batch upload: skipping '{filename}' — {e}") - failed.append(filename) - except Exception as e: - logger.error(f"Batch upload: unexpected error for '{filename}' — {e}") - failed.append(filename) - - if not created_documents and failed: - raise ValidationException(f"All files failed validation: {', '.join(failed)}") - - return BatchUploadResponse( - documents=created_documents, - task_ids=task_ids, - total=len(created_documents), - failed=failed, - ) - @router.post("/urlupload", status_code=status.HTTP_202_ACCEPTED) async def upload_document_url( payload: UploadUrl, - request: Request = None, background_tasks: BackgroundTasks = None, user: User = Depends(get_current_user), db: Session = Depends(get_db), @@ -437,29 +276,17 @@ async def upload_document_url( On Linux (production) a plain new_event_loop() is used instead. """ if CRAWL4AI_IMPORT_ERROR is not None: - raise ExternalServiceException("crawl4ai", "URL upload is unavailable because crawl4ai is not installed") + raise HTTPException( + status_code=503, + detail="URL upload is unavailable because crawl4ai is not installed", + ) temp_path: Optional[str] = None try: parsed = urlparse(payload.url) if not all([parsed.scheme, parsed.netloc]): - raise ValidationException("Invalid URL") - - # SSRF protection - BLOCKED_SCHEMES = {"file", "ftp", "gopher", "dict", "smb", "ldap"} - if parsed.scheme.lower() in BLOCKED_SCHEMES: - raise ValidationException(f"URL scheme '{parsed.scheme}' is not allowed") - if parsed.scheme.lower() not in ("http", "https"): - raise ValidationException("Only http and https URLs are allowed") - try: - hostname = parsed.hostname - if hostname: - addr = socket.getaddrinfo(hostname, 80)[0][4][0] - ip = ipaddress.ip_address(addr) - if ip.is_private or ip.is_loopback or ip.is_link_local: - raise ValidationException("Internal or private URLs are not allowed") - except (socket.gaierror, ValueError, IndexError): - raise ValidationException("Could not resolve URL host") + raise HTTPException(status_code=400, detail="Invalid URL") + # Run in a worker thread with its own event loop to avoid # NotImplementedError on Windows (SelectorEventLoop can't spawn subprocesses) @@ -469,7 +296,7 @@ async def upload_document_url( ) if not markdown: - raise ValidationException("No content could be extracted from the URL") + raise HTTPException(status_code=422, detail="No content could be extracted from the URL") with tempfile.NamedTemporaryFile( @@ -497,15 +324,6 @@ async def upload_document_url( url_path = parsed.path.rstrip("/") original_name = f"{parsed.netloc}{url_path or ''}.txt" - # Bind URL crawl metadata to request state and context variables - if request is not None: - request.state.filename = original_name - request.state.filesize = file_size - from app.observability import upload_filename_var, upload_filesize_var - upload_filename_var.set(original_name) - upload_filesize_var.set(file_size) - logger.info(f"URL crawler crawl completed, starting ingestion: {original_name} ({file_size} bytes)") - # ── Create database record ───────────────────────────── document = Document( user_id=user.id, @@ -542,56 +360,19 @@ async def upload_document_url( return DocumentResponse.model_validate(document).model_copy(update={"task_id": task_id}) - except AppException: + except HTTPException: raise except ValueError: - raise ValidationException("Invalid URL") + raise HTTPException(status_code=400, detail="Invalid URL") except Exception as e: logger.error(f"URL upload error: {e}") - raise ValidationException(f"Something went wrong with URL processing: {str(e)}") + raise HTTPException(status_code=400, detail=f"Something went wrong with URL processing: {str(e)}") finally: '''Runs whether the request succeeded, raised an HTTPException, or hit an unexpected error — no temp files are ever left behind.''' if temp_path is not None: Path(temp_path).unlink(missing_ok=True) -@router.get("/trash", response_model=DocumentListResponse) -def list_trash(user: User = Depends(get_current_user), db: Session = Depends(get_db)): - """ - List all soft-deleted documents for the authenticated user. - """ - docs = db.query(Document).filter( - Document.user_id == user.id, - Document.is_deleted.is_(True) - ).order_by(Document.deleted_at.desc()).all() - - return DocumentListResponse( - items=[DocumentResponse.model_validate(d) for d in docs], - total=len(docs), - page=1, - pages=1 - ) - -@router.post("/{document_id}/restore", response_model=DocumentResponse) -def restore_document(document_id: str, user: User = Depends(get_current_user), db: Session = Depends(get_db)): - """ - Restore a soft-deleted document. - """ - doc = db.query(Document).filter( - Document.id == document_id, - Document.user_id == user.id, - Document.is_deleted.is_(True) - ).first() - - if not doc: - raise HTTPException(status_code=404, detail="Document not found in trash") - - doc.is_deleted = False - doc.deleted_at = None - db.commit() - db.refresh(doc) - - return DocumentResponse.model_validate(doc) @router.get("/{document_id}/status", response_model=DocumentStatusResponse) @@ -614,90 +395,76 @@ def get_document_status( ).first() if not doc: - raise NotFoundException("Document") + raise HTTPException(status_code=404, detail="Document not found") return DocumentStatusResponse.model_validate(doc) @router.get("/", response_model=DocumentListResponse) def list_documents( - page: int = Query(1, ge=1, description="Page number (1-indexed)"), - per_page: int = Query(20, ge=1, le=100, description="Results per page"), - limit: int = Query(None, ge=1, le=100, description="Alias for per_page"), - q: Optional[str] = Query(None, description="Filter by document name (case-insensitive)"), - query: Optional[str] = Query(None, description="Alias for q – filter by document name"), + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1), user: User = Depends(get_current_user), db: Session = Depends(get_db), ): """ - List documents for the authenticated user with offset pagination and - optional keyword search on document names. + List all documents for the authenticated user with pagination. - Pagination is controlled by ``page`` and ``per_page`` (or its alias - ``limit``). Search is applied via ``query`` (or its short alias ``q``), - which performs a case-insensitive substring match on ``original_name``. + Returns a paginated list of documents belonging to the current user, + ordered by upload date (newest first). Args: - page: Page number to retrieve (1-indexed). Defaults to 1. - per_page: Number of documents per page. Defaults to 20, max 100. - limit: Alias for per_page – whichever is supplied takes effect. - q: Case-insensitive substring filter on original_name. - query: Alias for q – whichever is supplied takes effect. - user: Authenticated user injected by get_current_user. - db: Database session injected by get_db. - + page: The page number to retrieve (1: indexed). Defaults to 1. + per_page: The number of documents to return per page. Defaults to 20. + user: The currently authenticated user, injected by the `get_current_user` dependency. + db: Database session, injected by the `get_db` dependency. + Returns: - DocumentListResponse with items, total, page, pages, total_pages, - limit, and query fields. + DocumentListResponse: A response model containing: + - items: A list of DocumentResponse objects for the current page. + - total: The total number of documents for the user. + - page: The current page number. + - pages: The total number of pages available. """ - # Allow `limit` as alias for `per_page`; `query` as alias for `q` - effective_limit = limit if limit is not None else per_page - effective_query = query if query is not None else q - skip = (page - 1) * effective_limit - - # ── Build filtered query via helper ─────────────────────────────────────── - base_query = _get_documents_query(db, user.id, effective_query) - - # ── Total count (before pagination) ────────────────────────────────────── - total = db.execute( - select(func.count()).select_from(base_query.subquery()) - ).scalar_one() - # ── Paginated results ───────────────────────────────────────────────────── - docs = db.execute( - base_query - .order_by(Document.uploaded_at.desc()) - .limit(effective_limit) - .offset(skip) - ).scalars().all() + """Number of rows to skip""" + skip: int = (page - 1) * per_page - total_pages = max(1, (total + effective_limit - 1) // effective_limit) + """Total Pages""" + totalDocuments = ( + db.query(Document) + .filter(Document.user_id == user.id, Document.is_deleted.is_(False)) + .count() + ) + """Total Pages""" + pages = (totalDocuments + per_page - 1) // per_page + + """List all documents for the authenticated user in Paginated form""" + docs = (( + db.execute(select(Document) + .where(Document.user_id == user.id, Document.is_deleted.is_(False)) + .order_by(Document.uploaded_at.desc()) + .limit(per_page).offset(skip)) + ) + .scalars().all()) return DocumentListResponse( - items=[_deserialize_doc(d) for d in docs], - total=total, + items=[DocumentResponse.model_validate(d) for d in docs], + total=totalDocuments, page=page, - pages=total_pages, - total_pages=total_pages, - limit=effective_limit, - query=effective_query, + pages=pages ) @router.patch("/{document_id}", response_model=DocumentResponse) -def update_document( +def rename_document( document_id: str, - update: DocumentUpdate, + rename: DocumentRename, user: User = Depends(get_current_user), db: Session = Depends(get_db), ): """ - Update an uploaded document's metadata (name and/or summary) without - changing its stored file or vector data. - - Both fields are optional so callers can send a partial update. If a field - is not present (or is null) it will be left unchanged. To clear the - current summary send an explicit empty string. + Rename an uploaded document without changing its stored file or vector data. """ doc = db.query(Document).filter( Document.id == document_id, @@ -705,21 +472,16 @@ def update_document( ).first() if not doc: - raise NotFoundException("Document") + raise HTTPException(status_code=404, detail="Document not found") if str(doc.user_id) != str(user.id): - raise ForbiddenException("You do not have permission to update this document") - - if update.name is not None: - doc.original_name = update.name - if update.summary is not None: - stripped = update.summary.strip() - doc.summary = stripped if stripped else None + raise HTTPException(status_code=403, detail="You do not have permission to rename this document") + doc.original_name = rename.name db.commit() db.refresh(doc) - return _deserialize_doc(doc) + return DocumentResponse.model_validate(doc) @router.get("/{document_id}", response_model=DocumentResponse) @@ -753,9 +515,9 @@ def get_document( ).first() if not doc: - raise NotFoundException("Document") + raise HTTPException(status_code=404, detail="Document not found") - return _deserialize_doc(doc) + return DocumentResponse.model_validate(doc) @router.get("/{document_id}/pdf") @@ -790,12 +552,12 @@ def serve_pdf( ).first() if not doc: - raise NotFoundException("Document") + raise HTTPException(status_code=404, detail="Document not found") filepath = os.path.join(settings.UPLOAD_DIR, user.id, doc.filename) if not os.path.exists(filepath): - raise NotFoundException("File") + raise HTTPException(status_code=404, detail="File not found on disk") return FileResponse( filepath, @@ -840,7 +602,7 @@ def delete_document( ).first() if not doc: - raise NotFoundException("Document") + raise HTTPException(status_code=404, detail="Document not found") doc.is_deleted = True doc.deleted_at = datetime.now(timezone.utc) @@ -873,7 +635,6 @@ def update_chunk_settings( Raises: HTTPException: With status code 404 if the document is not found or does not belong to the authenticated user. - HTTPException: With status code 409 if the document is currently processing (re-chunking would race a concurrent ingestion run). HTTPException: With status code 400 if the provided chunk size or overlap values are invalid (e.g., chunk size less than 100, or overlap greater than or equal to chunk size). """ # Validate if the document exists and belongs to the user @@ -884,26 +645,15 @@ def update_chunk_settings( ).first() if not doc: - raise NotFoundException("Document") - - # Guard against re-queuing ingestion while a prior run for this same document is still actively processing. Without this, two - # process_document runs can execute concurrently against the same document_id: store_chunks() in vectorstore.py performs a non-atomic - # delete-then-batch-insert sequence, so one run's delete can fire in between the other run's delete-then-insert steps, corrupting the - # vector store and leaving Postgres' chunk_count out of sync with whatever vectors actually survive. - if doc.status == "processing": - raise ConflictException( - "Document is still processing. Wait for the current run to " - "finish before changing chunk settings." - ) - + raise HTTPException(status_code=404, detail="Document not found") + if settings_update.chunk_size is not None: if settings_update.chunk_size < 100: - raise ValidationException("Chunk size must be at least 100") + raise HTTPException(400, "Chunk size must be at least 100") doc.chunk_size = settings_update.chunk_size if settings_update.chunk_overlap is not None: - chunk_size_val = settings_update.chunk_size if settings_update.chunk_size is not None else (doc.chunk_size or settings.CHUNK_SIZE) - if settings_update.chunk_overlap >= chunk_size_val: - raise ValidationException("Chunk overlap cannot be greater than or equal to chunk size") + if settings_update.chunk_overlap >= settings_update.chunk_size: + raise HTTPException(400, "Chunk overlap cannot be greater than or equal to chunk size") doc.chunk_overlap = settings_update.chunk_overlap # Refresh the document record to update the chunk settings before re-ingestion @@ -941,64 +691,4 @@ def update_chunk_settings( task_id = f"local_{uuid.uuid4().hex}" # Return the updated document record with new chunk settings - return _deserialize_doc(doc).model_copy(update={"task_id": task_id}) - - -@router.post("/{document_id}/retry", response_model=DocumentResponse) -def retry_document_processing( - document_id: str, - background_tasks: BackgroundTasks = None, - user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - """Retry processing for a failed document. - - Resets the document status back to 'pending', clears error fields, - and re-queues the document for ingestion. - """ - doc = db.query(Document).filter( - Document.id == document_id, - Document.user_id == user.id, - Document.is_deleted.is_(False), - ).first() - - if not doc: - raise NotFoundException("Document") - - if doc.status != "failed": - raise ValidationException("Only failed documents can be retried") - - doc.status = "pending" - doc.processing_progress = 0 - doc.processing_stage = "queued" - doc.error_message = None - doc.last_error_traceback = None - doc.completed_at = None - doc.chunk_count = 0 - doc.page_count = 0 - db.commit() - - # Re-queue ingestion - filepath = os.path.join(settings.UPLOAD_DIR, user.id, doc.filename) - task_id = None - try: - task = process_document.delay( - document_id=doc.id, - filepath=filepath, - original_name=doc.original_name, - user_id=user.id, - ) - task_id = task.id - except Exception as e: - logger.warning(f"Celery queue failed for retry, falling back to background task: {e}") - if background_tasks: - background_tasks.add_task( - ingest_document, - document_id=doc.id, - filepath=filepath, - original_name=doc.original_name, - user_id=user.id, - ) - task_id = f"local_{uuid.uuid4().hex}" - - return _deserialize_doc(doc).model_copy(update={"task_id": task_id}) + return DocumentResponse.model_validate(doc).model_copy(update={"task_id": task_id}) diff --git a/backend/app/routes/graph.py b/backend/app/routes/graph.py deleted file mode 100644 index f8708634..00000000 --- a/backend/app/routes/graph.py +++ /dev/null @@ -1,175 +0,0 @@ -import logging -from pathlib import Path -from typing import Any, Dict, List, Optional - -from fastapi import APIRouter, Depends, HTTPException, status -from pydantic import BaseModel -from sqlalchemy.orm import Session - -from app.auth import get_current_user -from app.database import get_db -from app.models import Document, User -from app.rag.graph_builder import load_graph - -router = APIRouter(prefix="/graph", tags=["Knowledge Graph"]) -logger = logging.getLogger(__name__) - - -# ── Response schemas ────────────────────────────────────────────────────────── - -class GraphNode(BaseModel): - id: str - name: str - label: str # NER type, e.g. "PERSON", "ORG", "GPE" - mentions: int - pages: List[int] - - -class GraphEdge(BaseModel): - source: str - target: str - weight: int - pages: List[int] - - -class GraphResponse(BaseModel): - document_id: str - document_name: str - node_count: int - edge_count: int - nodes: List[GraphNode] - edges: List[GraphEdge] - - -class GraphSummaryResponse(BaseModel): - document_id: str - document_name: str - node_count: int - edge_count: int - top_entities: List[Dict[str, Any]] - graph_available: bool - - -# ── Helpers ─────────────────────────────────────────────────────────────────── - -def _get_owned_document(document_id: str, user: User, db: Session) -> Document: - """Return the document if it exists and belongs to the current user.""" - doc = ( - db.query(Document) - .filter(Document.id == document_id, Document.user_id == user.id) - .first() - ) - if not doc: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Document not found", - ) - return doc - - -# ── Routes ──────────────────────────────────────────────────────────────────── - -@router.get("/{document_id}", response_model=GraphResponse) -def get_graph( - document_id: str, - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - """ - Return the full knowledge graph (nodes + edges) for a document. - - The graph is built by ``graph_builder.py`` during ingestion and stored as - a JSON file on disk. If the graph file does not exist yet (document still - processing, or graph extraction was skipped) a 404 is returned so the - frontend can show an appropriate empty state. - """ - doc = _get_owned_document(document_id, current_user, db) - - graph = load_graph(str(current_user.id), document_id) - if graph is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Knowledge graph not available for this document yet. " - "The document may still be processing.", - ) - - nodes: List[GraphNode] = [] - for node_id, data in graph.nodes(data=True): - nodes.append( - GraphNode( - id=node_id, - name=data.get("name", node_id), - label=data.get("label", "UNKNOWN"), - mentions=data.get("mentions", 1), - pages=sorted(data.get("pages", [])), - ) - ) - - edges: List[GraphEdge] = [] - for source, target, data in graph.edges(data=True): - edges.append( - GraphEdge( - source=source, - target=target, - weight=data.get("weight", 1), - pages=sorted(data.get("pages", [])), - ) - ) - - return GraphResponse( - document_id=document_id, - document_name=doc.original_name, - node_count=graph.number_of_nodes(), - edge_count=graph.number_of_edges(), - nodes=nodes, - edges=edges, - ) - - -@router.get("/{document_id}/summary", response_model=GraphSummaryResponse) -def get_graph_summary( - document_id: str, - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - """ - Return lightweight graph statistics without the full node/edge payload. - - Useful for deciding whether to show the graph toggle button in the UI. - """ - doc = _get_owned_document(document_id, current_user, db) - - graph = load_graph(str(current_user.id), document_id) - if graph is None: - return GraphSummaryResponse( - document_id=document_id, - document_name=doc.original_name, - node_count=0, - edge_count=0, - top_entities=[], - graph_available=False, - ) - - # Top 10 nodes by mention count - top_entities = sorted( - [ - { - "id": node_id, - "name": data.get("name", node_id), - "label": data.get("label", "UNKNOWN"), - "mentions": data.get("mentions", 1), - } - for node_id, data in graph.nodes(data=True) - ], - key=lambda x: x["mentions"], - reverse=True, - )[:10] - - return GraphSummaryResponse( - document_id=document_id, - document_name=doc.original_name, - node_count=graph.number_of_nodes(), - edge_count=graph.number_of_edges(), - top_entities=top_entities, - graph_available=True, - ) diff --git a/backend/app/routes/health.py b/backend/app/routes/health.py deleted file mode 100644 index 14265f3d..00000000 --- a/backend/app/routes/health.py +++ /dev/null @@ -1,142 +0,0 @@ -""" -Deep health check endpoint — verifies DB, Redis, Celery, and ChromaDB. - -GET /api/v1/health/status -- No authentication required (monitoring tools must reach it freely). -- Never raises a 5xx — always returns 200 with per-component status so - load balancers and uptime monitors can parse the JSON body. -""" -import logging -import time -from typing import Dict, Any - -from fastapi import APIRouter -from fastapi.responses import JSONResponse -from sqlalchemy import text - -logger = logging.getLogger(__name__) - -router = APIRouter(prefix="/health", tags=["Health"]) - - -def _check_database() -> Dict[str, Any]: - """Ping the configured database (SQLite or PostgreSQL).""" - start = time.perf_counter() - try: - from app.database import engine - with engine.connect() as conn: - conn.execute(text("SELECT 1")) - latency_ms = round((time.perf_counter() - start) * 1000, 2) - return {"status": "up", "latency_ms": latency_ms} - except Exception as exc: - logger.warning("DB health check failed: %s", exc) - return {"status": "down", "error": str(exc)[:120]} - - -def _check_redis() -> Dict[str, Any]: - """Ping the Redis broker used by Celery and the response cache.""" - start = time.perf_counter() - try: - from app.cache import _get_redis - r = _get_redis() - if r is None: - return {"status": "unavailable", "detail": "REDIS_URL not configured"} - r.ping() - latency_ms = round((time.perf_counter() - start) * 1000, 2) - return {"status": "up", "latency_ms": latency_ms} - except Exception as exc: - logger.warning("Redis health check failed: %s", exc) - return {"status": "down", "error": str(exc)[:120]} - - -def _check_celery() -> Dict[str, Any]: - """ - Verify at least one Celery worker is reachable via the inspect API. - Uses a short timeout so the endpoint stays fast even when workers - are offline. - """ - start = time.perf_counter() - try: - from app.celery_app import celery_app - - inspector = celery_app.control.inspect(timeout=2.0) - active = inspector.active() # None if no workers respond - ping = inspector.ping() - - latency_ms = round((time.perf_counter() - start) * 1000, 2) - - if not ping: - return { - "status": "down", - "detail": "No Celery workers responded to ping", - "latency_ms": latency_ms, - } - - worker_names = list(ping.keys()) - active_task_count = sum( - len(tasks) for tasks in (active or {}).values() - ) - return { - "status": "up", - "workers": len(worker_names), - "worker_names": worker_names, - "active_tasks": active_task_count, - "latency_ms": latency_ms, - } - except Exception as exc: - logger.warning("Celery health check failed: %s", exc) - return {"status": "down", "error": str(exc)[:120]} - - -def _check_chromadb() -> Dict[str, Any]: - """Ping the ChromaDB vector store.""" - start = time.perf_counter() - try: - from app.rag.vectorstore import get_chroma_client - client = get_chroma_client() - client.heartbeat() - latency_ms = round((time.perf_counter() - start) * 1000, 2) - return {"status": "up", "latency_ms": latency_ms} - except Exception as exc: - logger.warning("ChromaDB health check failed: %s", exc) - return {"status": "down", "error": str(exc)[:120]} - - -@router.get( - "/status", - summary="Deep system health check", - description=( - "Returns connection status for PostgreSQL/SQLite, Redis, " - "Celery workers, and ChromaDB. Always returns HTTP 200 — " - "inspect the `overall` field to determine system health." - ), -) -def health_status(): - """ - Deep health check — verifies all backend dependencies. - - Returns HTTP 200 always. Callers should inspect the ``overall`` field: - - ``healthy`` — all components up. - - ``degraded`` — one or more components down or unavailable. - """ - checks = { - "database": _check_database(), - "redis": _check_redis(), - "celery": _check_celery(), - "chromadb": _check_chromadb(), - } - - # overall is healthy only when every component is "up" - all_up = all( - v.get("status") == "up" - for v in checks.values() - ) - overall = "healthy" if all_up else "degraded" - - return JSONResponse( - status_code=200, - content={ - "overall": overall, - "components": checks, - }, - ) diff --git a/backend/app/routes/profile.py b/backend/app/routes/profile.py index 365ec11e..7fea15f7 100644 --- a/backend/app/routes/profile.py +++ b/backend/app/routes/profile.py @@ -1,12 +1,10 @@ -from fastapi import APIRouter, Depends, UploadFile, File +from fastapi import APIRouter, Depends, UploadFile, File, HTTPException from sqlalchemy.orm import Session -from sqlalchemy.exc import IntegrityError from pathlib import Path import shutil import uuid from app.database import get_db -from app.exceptions import ValidationException, NotFoundException from app.models import User from app.schemas import UserProfileUpdate, UserResponse from app.auth import get_current_user @@ -29,27 +27,12 @@ def update_profile( current_user: User = Depends(get_current_user), ): if payload.username: - existing_user = ( - db.query(User) - .filter( - User.username == payload.username, - User.id != current_user.id, - ) - .first() - ) - if existing_user: - raise ValidationException("Username already exists") current_user.username = payload.username if payload.display_name: current_user.display_name = payload.display_name - try: - db.commit() - except IntegrityError: - db.rollback() - raise ValidationException("Username already exists") - + db.commit() db.refresh(current_user) return current_user @@ -66,7 +49,7 @@ def upload_avatar( extension = Path(file.filename).suffix.lower() if extension not in allowed_extensions: - raise ValidationException("Invalid image format") + raise HTTPException(status_code=400, detail="Invalid image format") filename = f"{uuid.uuid4()}{extension}" filepath = UPLOAD_DIR / filename diff --git a/backend/app/routes/workspaces.py b/backend/app/routes/workspaces.py index 50ce268a..7ce2de63 100644 --- a/backend/app/routes/workspaces.py +++ b/backend/app/routes/workspaces.py @@ -1,4 +1,4 @@ -"""Workspace invitation and management routes.""" +"""Workspace invitation routes for admin-managed workspace access.""" import hashlib import logging @@ -7,76 +7,19 @@ from fastapi import APIRouter, Depends, HTTPException, status from sqlalchemy.orm import Session -from sqlalchemy import select -from typing import List -from app.auth import create_invite_token, get_admin_user, get_current_user +from app.auth import create_invite_token, get_admin_user from app.config import get_settings from app.database import get_db -from app.email_service import send_workspace_invite_email -from app.exceptions import ( - ConflictException, - ForbiddenException, - NotFoundException, - ValidationException, -) -from app.models import User, Workspace, WorkspaceInvitation, WorkspaceMember, WorkspaceRole -from app.schemas import ( - WorkspaceCreate, - WorkspaceDetailResponse, - WorkspaceInviteRequest, - WorkspaceInviteResponse, - WorkspaceMemberAdd, - WorkspaceMemberResponse, - WorkspaceMemberRoleUpdate, - WorkspaceResponse, -) +from app.email_service import send_email +from app.models import User, WorkspaceInvitation +from app.schemas import WorkspaceInviteRequest, WorkspaceInviteResponse router = APIRouter(prefix="/workspaces", tags=["Workspaces"]) settings = get_settings() logger = logging.getLogger(__name__) -# ── Private helpers ─────────────────────────────────────────────────────────── - -def _get_workspace_or_404(workspace_id: str, db: Session) -> Workspace: - ws = db.get(Workspace, workspace_id) - if not ws: - raise NotFoundException("Workspace") - return ws - - -def _get_membership_or_403(workspace: Workspace, user: User, db: Session) -> WorkspaceMember: - """Return the caller's WorkspaceMember row, or raise 403.""" - membership = db.execute( - select(WorkspaceMember).where( - WorkspaceMember.workspace_id == workspace.id, - WorkspaceMember.user_id == user.id, - ) - ).scalar_one_or_none() - if not membership: - raise ForbiddenException("You are not a member of this workspace") - return membership - - -def _require_workspace_admin(membership: WorkspaceMember) -> None: - if membership.role != WorkspaceRole.admin: - raise ForbiddenException("Only workspace admins can perform this action") - - -def _count_admins(workspace_id: str, db: Session) -> int: - return len( - db.execute( - select(WorkspaceMember).where( - WorkspaceMember.workspace_id == workspace_id, - WorkspaceMember.role == WorkspaceRole.admin, - ) - ).scalars().all() - ) - - -# ── Invitation (existing) ───────────────────────────────────────────────────── - @router.post("/invite", response_model=WorkspaceInviteResponse, status_code=status.HTTP_200_OK) def invite_workspace( payload: WorkspaceInviteRequest, @@ -107,13 +50,20 @@ def invite_workspace( db.refresh(invitation) join_link = f"{settings.APP_URL.rstrip('/')}/invite?token={quote(token, safe='')}" - send_workspace_invite_email( - to=payload.email, - workspace_name=payload.workspace_name, - invite_link=join_link, - expires_in_hours=settings.INVITE_TOKEN_EXPIRY_HOURS, - personal_message=payload.message or None, - ) + subject = f"Invitation to join workspace '{payload.workspace_name}'" + body_lines = [ + f"Hello,", + "", + f"You have been invited to join the workspace '{payload.workspace_name}'.", + "Click the link below to accept the invitation:", + join_link, + ] + if payload.message: + body_lines.insert(3, payload.message) + body_lines.insert(4, "") + body = "\n".join(body_lines) + + send_email(payload.email, subject, body) return WorkspaceInviteResponse( email=payload.email, @@ -121,310 +71,3 @@ def invite_workspace( invite_link=join_link, expires_in_hours=settings.INVITE_TOKEN_EXPIRY_HOURS, ) - - -# ── Workspace CRUD ──────────────────────────────────────────────────────────── - -@router.post( - "", - response_model=WorkspaceDetailResponse, - status_code=status.HTTP_201_CREATED, - summary="Create a workspace", -) -def create_workspace( - payload: WorkspaceCreate, - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - """ - Create a new workspace. - - The authenticated user is automatically added as the first member with - the ``admin`` role. - """ - workspace = Workspace( - name=payload.name, - created_by=current_user.id, - ) - db.add(workspace) - db.flush() # populate workspace.id before creating the membership row - - membership = WorkspaceMember( - workspace_id=workspace.id, - user_id=current_user.id, - role=WorkspaceRole.admin, - ) - db.add(membership) - db.commit() - db.refresh(workspace) - - logger.info( - "Workspace '%s' (%s) created by user %s", - workspace.name, - workspace.id, - current_user.id, - ) - return workspace - - -@router.get( - "", - response_model=List[WorkspaceResponse], - summary="List my workspaces", -) -def list_workspaces( - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - """Return every workspace the authenticated user is a member of.""" - memberships = db.execute( - select(WorkspaceMember).where(WorkspaceMember.user_id == current_user.id) - ).scalars().all() - - workspace_ids = [m.workspace_id for m in memberships] - if not workspace_ids: - return [] - - return db.execute( - select(Workspace).where(Workspace.id.in_(workspace_ids)) - ).scalars().all() - - -@router.get( - "/{workspace_id}", - response_model=WorkspaceDetailResponse, - summary="Get workspace detail", -) -def get_workspace( - workspace_id: str, - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - """ - Return workspace details including the full member list. - - Only workspace members may access this endpoint. - """ - workspace = _get_workspace_or_404(workspace_id, db) - _get_membership_or_403(workspace, current_user, db) - return workspace - - -@router.delete( - "/{workspace_id}", - status_code=status.HTTP_204_NO_CONTENT, - summary="Delete a workspace", -) -def delete_workspace( - workspace_id: str, - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - """ - Permanently delete a workspace and all its members. - - Requires the caller to hold the workspace ``admin`` role. - """ - workspace = _get_workspace_or_404(workspace_id, db) - membership = _get_membership_or_403(workspace, current_user, db) - _require_workspace_admin(membership) - - db.delete(workspace) - db.commit() - - logger.info( - "Workspace '%s' (%s) deleted by user %s", - workspace.name, - workspace_id, - current_user.id, - ) - return None - - -# ── Member management ───────────────────────────────────────────────────────── - -@router.post( - "/{workspace_id}/members", - response_model=WorkspaceMemberResponse, - status_code=status.HTTP_201_CREATED, - summary="Add a member to a workspace", -) -def add_member( - workspace_id: str, - payload: WorkspaceMemberAdd, - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - """ - Add an existing user to a workspace. - - * Requires the caller to be a workspace ``admin``. - * The target user must already exist in the system (looked up by ``user_id``). - * Adding an existing member raises ``409 Conflict``. - * Defaults to the ``viewer`` role when no role is supplied. - """ - workspace = _get_workspace_or_404(workspace_id, db) - caller_membership = _get_membership_or_403(workspace, current_user, db) - _require_workspace_admin(caller_membership) - - target_user = db.get(User, payload.user_id) - if not target_user: - raise NotFoundException("User") - - existing = db.execute( - select(WorkspaceMember).where( - WorkspaceMember.workspace_id == workspace_id, - WorkspaceMember.user_id == payload.user_id, - ) - ).scalar_one_or_none() - if existing: - raise ConflictException("User is already a member of this workspace") - - new_membership = WorkspaceMember( - workspace_id=workspace_id, - user_id=payload.user_id, - role=payload.role, - ) - db.add(new_membership) - db.commit() - db.refresh(new_membership) - - logger.info( - "User %s added to workspace %s as '%s' by %s", - payload.user_id, - workspace_id, - payload.role, - current_user.id, - ) - return new_membership - - -@router.get( - "/{workspace_id}/members", - response_model=List[WorkspaceMemberResponse], - summary="List workspace members", -) -def list_members( - workspace_id: str, - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - """ - Return all members of a workspace. - - Only workspace members may call this endpoint. - """ - workspace = _get_workspace_or_404(workspace_id, db) - _get_membership_or_403(workspace, current_user, db) - - return db.execute( - select(WorkspaceMember).where(WorkspaceMember.workspace_id == workspace_id) - ).scalars().all() - - -@router.patch( - "/{workspace_id}/members/{user_id}", - response_model=WorkspaceMemberResponse, - summary="Change a member's role", -) -def update_member_role( - workspace_id: str, - user_id: str, - payload: WorkspaceMemberRoleUpdate, - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - """ - Change a member's role within the workspace. - - * Requires the caller to be a workspace ``admin``. - * Demoting the last ``admin`` is rejected to prevent lockout. - """ - workspace = _get_workspace_or_404(workspace_id, db) - caller_membership = _get_membership_or_403(workspace, current_user, db) - _require_workspace_admin(caller_membership) - - target_membership = db.execute( - select(WorkspaceMember).where( - WorkspaceMember.workspace_id == workspace_id, - WorkspaceMember.user_id == user_id, - ) - ).scalar_one_or_none() - if not target_membership: - raise NotFoundException("Workspace member") - - if ( - target_membership.role == WorkspaceRole.admin - and payload.role != WorkspaceRole.admin - and _count_admins(workspace_id, db) <= 1 - ): - raise ValidationException( - "Cannot demote the last admin. Promote another member to admin first." - ) - - target_membership.role = payload.role - db.commit() - db.refresh(target_membership) - - logger.info( - "User %s role in workspace %s changed to '%s' by %s", - user_id, - workspace_id, - payload.role, - current_user.id, - ) - return target_membership - - -@router.delete( - "/{workspace_id}/members/{user_id}", - status_code=status.HTTP_204_NO_CONTENT, - summary="Remove a member from a workspace", -) -def remove_member( - workspace_id: str, - user_id: str, - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db), -): - """ - Remove a member from a workspace. - - * A workspace ``admin`` may remove any member. - * Any member may remove themselves (leave the workspace). - * Removing the last ``admin`` is rejected to prevent lockout. - """ - workspace = _get_workspace_or_404(workspace_id, db) - caller_membership = _get_membership_or_403(workspace, current_user, db) - - is_self = str(current_user.id) == str(user_id) - if not is_self: - _require_workspace_admin(caller_membership) - - target_membership = db.execute( - select(WorkspaceMember).where( - WorkspaceMember.workspace_id == workspace_id, - WorkspaceMember.user_id == user_id, - ) - ).scalar_one_or_none() - if not target_membership: - raise NotFoundException("Workspace member") - - if ( - target_membership.role == WorkspaceRole.admin - and _count_admins(workspace_id, db) <= 1 - ): - raise ValidationException( - "Cannot remove the last admin. Promote another member to admin first." - ) - - db.delete(target_membership) - db.commit() - - logger.info( - "User %s removed from workspace %s by %s", - user_id, - workspace_id, - current_user.id, - ) - return None diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index 2da9afea..516029e3 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -44,48 +44,6 @@ def start_scheduler(): else: logger.info("Drive PDF sync disabled") - # Document processing recovery — every 5 minutes - try: - from app.services.cleanup import cleanup_stale_documents, cleanup_old_deleted_documents, cleanup_inactive_active_documents - - _scheduler.add_job( - cleanup_stale_documents, - trigger=IntervalTrigger(minutes=5), - id="recover_stale_processing", - name="Recover documents stuck in processing", - replace_existing=True, - max_instances=1, - coalesce=True, - misfire_grace_time=60, - ) - logger.info("Stale document recovery scheduled every 5 minutes") - - _scheduler.add_job( - cleanup_old_deleted_documents, - trigger=IntervalTrigger(days=1), - id="cleanup_old_deleted", - name="Purge old soft-deleted documents", - replace_existing=True, - max_instances=1, - coalesce=True, - misfire_grace_time=300, - ) - logger.info("Old deleted document cleanup scheduled daily") - - _scheduler.add_job( - cleanup_inactive_active_documents, - trigger=IntervalTrigger(days=1), - id="cleanup_inactive_active", - name="Purge active documents inactive beyond threshold", - replace_existing=True, - max_instances=1, - coalesce=True, - misfire_grace_time=300, - ) - logger.info("Inactive active document cleanup scheduled daily") - except Exception as e: - logger.warning("Could not schedule cleanup jobs: %s", e) - _scheduler.start() return _scheduler diff --git a/backend/app/schemas.py b/backend/app/schemas.py index 8ca7441d..f6c0c752 100644 --- a/backend/app/schemas.py +++ b/backend/app/schemas.py @@ -1,35 +1,13 @@ """ Pydantic schemas for API request/response validation. """ -import json -import re from pydantic import BaseModel, EmailStr, Field, field_validator -from typing import Optional, List, Any +from typing import Optional, List from datetime import datetime -from app.models import UserRole, WorkspaceRole +from app.models import UserRole from app.password_validation import validate_password -class ErrorDetail(BaseModel): - field: str - message: str - - -class ErrorEnvelope(BaseModel): - code: str - message: str - details: dict[str, Any] = {} - request_id: str | None = None - - -class ErrorResponse(BaseModel): - error: ErrorEnvelope - - -class ValidationErrorResponse(BaseModel): - error: ErrorEnvelope - - # ── Auth ───────────────────────────────────────────── class UserRegister(BaseModel): @@ -94,16 +72,6 @@ class UpdatePasswordResponse(BaseModel): email: EmailStr password_changed: bool = True -class ChangePasswordRequest(BaseModel): - current_password: str - new_password: str = Field(..., min_length=8) - - @field_validator("new_password") - @classmethod - def validate_password_strength(cls, value: str) -> str: - validate_password(value) - return value - class WorkspaceInviteRequest(BaseModel): email: EmailStr @@ -130,7 +98,8 @@ class RefreshRequest(BaseModel): class HFTokenUpdate(BaseModel): - hf_token: str = Field(..., min_length=1, max_length=500) + """Request schema for updating the user's HuggingFace token.""" + hf_token: str class GoogleDriveAuthUrlResponse(BaseModel): @@ -191,30 +160,10 @@ class DocumentResponse(BaseModel): uploaded_at: datetime summary: Optional[str] = None # New field for document summary task_id: Optional[str] = None - keywords: Optional[List[str]] = [] - extracted_urls: Optional[List[str]] = None - - @field_validator("keywords", mode="before") - @classmethod - def parse_keywords(cls, v): - if v is None: - return [] - if isinstance(v, list): - return v - try: - return json.loads(v) - except (ValueError, TypeError): - return [] - class Config: from_attributes = True -class BatchUploadResponse(BaseModel): - documents: List[DocumentResponse] - task_ids: List[str] - total: int - failed: List[str] = [] class DocumentRename(BaseModel): name: str = Field(..., min_length=1, max_length=255) @@ -228,36 +177,12 @@ def validate_name(cls, value: str) -> str: return stripped -class DocumentUpdate(BaseModel): - """Schema for updating document metadata via PATCH. All fields are optional - so that callers can send a partial update (e.g. only the name or only the - summary) without having to include every field.""" - name: Optional[str] = Field(None, min_length=1, max_length=255) - summary: Optional[str] = Field(None, max_length=5000) - - @field_validator("name") - @classmethod - def validate_name(cls, value: Optional[str]) -> Optional[str]: - if value is not None: - stripped = value.strip() - if not stripped: - raise ValueError("Document name cannot be empty") - return stripped - return value - - class DocumentStatusResponse(BaseModel): id: str status: str page_count: int chunk_count: int error_message: Optional[str] = None - processing_progress: Optional[int] = None - processing_stage: Optional[str] = None - retry_count: Optional[int] = None - last_error_traceback: Optional[str] = None - processing_started_at: Optional[datetime] = None - completed_at: Optional[datetime] = None class Config: from_attributes = True @@ -268,9 +193,6 @@ class DocumentListResponse(BaseModel): total: int page: int pages: int - total_pages: int - limit: int - query: Optional[str] = None # Admin @@ -303,16 +225,6 @@ class ChatRequest(BaseModel): session_id: Optional[str] = None top_k: int = Field(default=5, ge=1, le=20) - @field_validator("question") - @classmethod - def sanitize_question(cls, v: str) -> str: - """Strip control characters (null bytes, ANSI escapes, etc.) from user input.""" - cleaned = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]", "", v) - stripped = cleaned.strip() - if not stripped: - raise ValueError("Question cannot be empty after sanitization") - return stripped - class SourceChunk(BaseModel): text: str @@ -351,16 +263,9 @@ class ChatHistoryResponse(BaseModel): # Chunk settings schema for optional chunk size and overlap parameters in document processing class ChunkSettings(BaseModel): - chunk_size: int = Field(default=1000, ge=100, le=2000) - chunk_overlap: int = Field(default=200, ge=0) - - @field_validator("chunk_overlap") - @classmethod - def validate_overlap(cls, v: int, info: Any) -> int: - if "chunk_size" in info.data and v >= info.data["chunk_size"]: - raise ValueError("chunk_overlap must be less than chunk_size") - return v - + chunk_size: int | None + chunk_overlap: int | None + class UploadUrl(BaseModel): url: str @@ -377,7 +282,7 @@ class ShareLinkResponse(BaseModel): class FeedbackRequest(BaseModel): - feedback: Optional[str] = Field(None, pattern="^(up|down)?$") + feedback: Optional[str] = None # ── Chat Session ────────────────────────────────────── @@ -385,9 +290,6 @@ class FeedbackRequest(BaseModel): class ChatSessionCreate(BaseModel): title: str = Field(..., min_length=1, max_length=255) -class ChatSessionUpdate(BaseModel): - title: str = Field(..., min_length=1, max_length=255) - class ChatSessionResponse(BaseModel): id: str @@ -398,53 +300,5 @@ class Config: from_attributes = True -# ── Workspaces ──────────────────────────────────────── - -class WorkspaceCreate(BaseModel): - name: str = Field(..., min_length=1, max_length=255) - - -class WorkspaceMemberResponse(BaseModel): - id: str - workspace_id: str - user_id: str - role: WorkspaceRole - joined_at: datetime - - class Config: - from_attributes = True - - -class WorkspaceResponse(BaseModel): - id: str - name: str - created_by: str - created_at: datetime - - class Config: - from_attributes = True - - -class WorkspaceDetailResponse(BaseModel): - """Workspace detail including the full member list.""" - id: str - name: str - created_by: str - created_at: datetime - members: List[WorkspaceMemberResponse] = [] - - class Config: - from_attributes = True - - -class WorkspaceMemberAdd(BaseModel): - user_id: str = Field(..., min_length=1) - role: WorkspaceRole = WorkspaceRole.viewer - - -class WorkspaceMemberRoleUpdate(BaseModel): - role: WorkspaceRole - - # Rebuild models for forward references TokenResponse.model_rebuild() diff --git a/backend/app/security.py b/backend/app/security.py deleted file mode 100644 index dddd306e..00000000 --- a/backend/app/security.py +++ /dev/null @@ -1,19 +0,0 @@ -# Create this new file or append to your existing helpers -from pathlib import Path -from fastapi import HTTPException, status - -def verify_secure_sandbox_path(filename: str, sandbox_base_dir: str) -> Path: - """ - Validates that a requested filename stays strictly within the sandbox directory. - Raises a clean 403 Forbidden error if a path traversal attempt is detected. - """ - base_path = Path(sandbox_base_dir).resolve() - target_path = (base_path / filename).resolve() - - if not target_path.is_relative_to(base_path): - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="Access Denied: Invalid resource path mapping." - ) - - return target_path diff --git a/backend/app/services/cleanup.py b/backend/app/services/cleanup.py deleted file mode 100644 index 15daf5ae..00000000 --- a/backend/app/services/cleanup.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Background cleanup jobs for stale, inactive, and deleted documents. - -All cleanup functions use batched pagination and transaction-safe patterns -to avoid race conditions, double-deletes, and database lock contention. -""" -import logging -from datetime import datetime, timedelta, timezone - -from app.database import get_db_session -from app.config import get_settings -from app.models import Document - -logger = logging.getLogger(__name__) -settings = get_settings() - -_CLEANUP_BATCH_SIZE = 100 - - -def _hard_delete_document(db, doc): - """Perform the three-step permanent deletion for a single document. - - Wrapped in try/except so that a failure on one document does not block - the remaining documents in the batch. - """ - doc_id = doc.id - user_id = doc.user_id - name = doc.original_name - - try: - from app.rag.vectorstore import delete_document_chunks - delete_document_chunks(document_id=doc_id, user_id=user_id) - except Exception as e: - logger.warning("Cleanup: error deleting vectors for %s: %s", doc_id, e) - - try: - import os - filepath = os.path.join(settings.UPLOAD_DIR, user_id, doc.filename) - if os.path.exists(filepath): - os.remove(filepath) - except Exception as e: - logger.warning("Cleanup: error deleting file for %s: %s", doc_id, e) - - try: - db.delete(doc) - except Exception as e: - logger.warning("Cleanup: error deleting DB record for %s: %s", doc_id, e) - - -def cleanup_stale_documents(): - """Mark documents stuck in 'processing' beyond the timeout as failed.""" - timeout_minutes = settings.DOC_PROCESSING_TIMEOUT_MINUTES - cutoff = datetime.now(timezone.utc) - timedelta(minutes=timeout_minutes) - - with get_db_session() as db: - offset = 0 - while True: - batch = ( - db.query(Document) - .filter( - Document.status == "processing", - Document.processing_started_at.isnot(None), - Document.processing_started_at < cutoff, - Document.is_deleted.is_(False), - ) - .limit(_CLEANUP_BATCH_SIZE) - .offset(offset) - .all() - ) - if not batch: - break - for doc in batch: - logger.warning( - "Recovering stale document %s (stuck at '%s' since %s)", - doc.id, - doc.processing_stage, - doc.processing_started_at, - ) - doc.status = "failed" - doc.processing_progress = 0 - doc.error_message = f"Processing timed out after {timeout_minutes} minutes" - doc.last_error_traceback = "Timed out: no progress update received within the configured timeout window." - logger.info("Marked stale document %s as failed", doc.id) - offset += _CLEANUP_BATCH_SIZE - - -def cleanup_old_deleted_documents(): - """Permanently delete documents soft-deleted beyond the max age.""" - max_age_days = settings.DOC_CLEANUP_MAX_AGE_DAYS - cutoff = datetime.now(timezone.utc) - timedelta(days=max_age_days) - - with get_db_session() as db: - offset = 0 - while True: - batch = ( - db.query(Document) - .filter( - Document.is_deleted.is_(True), - Document.deleted_at.isnot(None), - Document.deleted_at < cutoff, - ) - .limit(_CLEANUP_BATCH_SIZE) - .offset(offset) - .all() - ) - if not batch: - break - for doc in batch: - logger.info( - "Purging old deleted document %s ('%s', deleted %s)", - doc.id, - doc.original_name, - doc.deleted_at, - ) - _hard_delete_document(db, doc) - logger.info( - "Permanently deleted old document %s ('%s')", - doc.id, - doc.original_name, - ) - offset += _CLEANUP_BATCH_SIZE - - -def cleanup_inactive_active_documents(): - """Hard-delete active documents that have not been accessed for the - configured inactivity period. - - Only runs when ``DOC_CLEANUP_ENABLED`` is ``True``. - Uses batched pagination and filters out soft-deleted records to avoid - overlap with ``cleanup_old_deleted_documents``. - """ - if not settings.DOC_CLEANUP_ENABLED: - logger.info("Inactive document cleanup is disabled via DOC_CLEANUP_ENABLED") - return - - inactive_days = settings.DOC_CLEANUP_INACTIVE_DAYS - cutoff = datetime.now(timezone.utc) - timedelta(days=inactive_days) - - with get_db_session() as db: - offset = 0 - while True: - batch = ( - db.query(Document) - .filter( - Document.is_deleted.is_(False), - Document.last_accessed_at < cutoff, - ) - .limit(_CLEANUP_BATCH_SIZE) - .offset(offset) - .all() - ) - if not batch: - break - for doc in batch: - logger.info( - "Purging inactive active document %s ('%s', last accessed %s)", - doc.id, - doc.original_name, - doc.last_accessed_at, - ) - _hard_delete_document(db, doc) - logger.info( - "Purged inactive active document %s ('%s')", - doc.id, - doc.original_name, - ) - offset += _CLEANUP_BATCH_SIZE diff --git a/backend/app/services/document_ingestion.py b/backend/app/services/document_ingestion.py index 6e76d79d..b8bc65d6 100644 --- a/backend/app/services/document_ingestion.py +++ b/backend/app/services/document_ingestion.py @@ -1,35 +1,11 @@ """Reusable document ingestion pipeline.""" -import traceback import logging -from datetime import datetime, timezone from app.models import Document -from app.rag.agent import persist_document_keywords from app.rag.chunker import chunk_document, get_page_count from app.rag.vectorstore import store_chunks -from app.config import get_settings logger = logging.getLogger(__name__) -settings = get_settings() - - -def _update_progress(document_id: str, progress: int, stage: str, error: str = None): - """Update document progress fields in the database.""" - from app.database import SessionLocal - - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == document_id).first() - if doc: - doc.processing_progress = progress - doc.processing_stage = stage - if error: - doc.error_message = error - db.commit() - except Exception as e: - logger.warning("Failed to update progress for %s: %s", document_id, e) - finally: - db.close() def ingest_document(document_id: str, filepath: str, original_name: str, user_id: str): @@ -50,16 +26,11 @@ def ingest_document(document_id: str, filepath: str, original_name: str, user_id return doc.status = "processing" - doc.processing_stage = "extracting" - doc.processing_progress = 10 doc.error_message = None - doc.last_error_traceback = None db.commit() page_count = get_page_count(filepath) doc.page_count = page_count - doc.processing_progress = 20 - db.commit() try: chunk_kwargs = {} @@ -67,56 +38,17 @@ def ingest_document(document_id: str, filepath: str, original_name: str, user_id chunk_kwargs["chunk_size"] = doc.chunk_size if doc.chunk_overlap is not None: chunk_kwargs["chunk_overlap"] = doc.chunk_overlap - doc.processing_stage = "chunking" - doc.processing_progress = 30 - db.commit() chunks = chunk_document(filepath, **chunk_kwargs) except TypeError: + # Preserve compatibility with patched/test implementations. chunks = chunk_document(filepath) - # ── Proximity caption pass (PDF only) ──────────────────────────────── - # Write bounding-box-derived captions into image chunks BEFORE store_chunks() - # so generate_captions_for_chunks() in vectorstore.py only needs to handle - # the OCR / placeholder fallback for any images without adjacent text. - ext = filepath.rsplit(".", 1)[-1].lower() - if ext == "pdf": - try: - from app.rag.vision import extract_captions_from_pdf - - pdf_captions = extract_captions_from_pdf(filepath) - # Build lookup: page -> [captions in figure_index order] - caption_map: dict = {} - for cap in pdf_captions: - caption_map.setdefault(cap["page"], []).append(cap) - - fig_counters: dict = {} - for chunk in chunks: - if not chunk.get("image_bytes"): - continue - page = chunk.get("page", 1) - idx = fig_counters.get(page, 0) - page_caps = caption_map.get(page, []) - if idx < len(page_caps) and page_caps[idx]["caption"]: - chunk["image_caption"] = page_caps[idx]["caption"] - chunk["bbox"] = str(page_caps[idx]["bbox"]) - fig_counters[page] = idx + 1 - except Exception as exc: - logger.warning( - "Proximity caption extraction failed for %s: %s", document_id, exc - ) - # ── End proximity caption pass ──────────────────────────────────────── - if not chunks: doc.status = "failed" - doc.processing_progress = 0 doc.error_message = "No text could be extracted from the document" db.commit() return - doc.processing_progress = 50 - doc.processing_stage = "indexing" - db.commit() - try: from app.rag.graph_builder import build_graph, save_graph @@ -125,10 +57,6 @@ def ingest_document(document_id: str, filepath: str, original_name: str, user_id except Exception as e: logger.warning("Could not build knowledge graph for document %s: %s", document_id, e) - doc.processing_progress = 70 - doc.processing_stage = "embedding" - db.commit() - chunk_count = store_chunks( chunks=chunks, document_id=document_id, @@ -136,11 +64,6 @@ def ingest_document(document_id: str, filepath: str, original_name: str, user_id user_id=user_id, ) - persist_document_keywords(doc, chunks, db) - - doc.processing_progress = 85 - db.commit() - try: from app.rag.summarizer import generate_document_summary @@ -152,34 +75,8 @@ def ingest_document(document_id: str, filepath: str, original_name: str, user_id logger.warning("Could not generate summary for document %s: %s", document_id, e) doc.summary = None - # ── URL extraction pass (PDF only) ──────────────────────────────── - ext = filepath.rsplit(".", 1)[-1].lower() - if ext == "pdf": - try: - from app.rag.url_extractor import extract_urls_from_pdf - import json - - urls = extract_urls_from_pdf(filepath) - doc.extracted_urls = json.dumps(urls) if urls else None - db.commit() - logger.info( - "Extracted %s URLs from document %s", - len(urls), - document_id, - ) - except Exception as exc: - logger.warning( - "URL extraction failed for document %s: %s", - document_id, - exc, - ) - # ── End URL extraction pass ─────────────────────────────────────── - doc.chunk_count = chunk_count doc.status = "ready" - doc.processing_progress = 100 - doc.processing_stage = "completed" - doc.completed_at = datetime.now(timezone.utc) doc.error_message = None db.commit() @@ -192,7 +89,6 @@ def ingest_document(document_id: str, filepath: str, original_name: str, user_id except Exception as e: logger.error("Ingestion error for %s: %s", document_id, e) - db.rollback() try: doc = db.query(Document).filter( Document.id == document_id, @@ -200,9 +96,7 @@ def ingest_document(document_id: str, filepath: str, original_name: str, user_id ).first() if doc: doc.status = "failed" - doc.processing_progress = 0 doc.error_message = str(e)[:500] - doc.last_error_traceback = traceback.format_exc()[:2000] db.commit() except Exception: logger.exception("Failed to mark document %s as failed", document_id) diff --git a/backend/app/services/layout_parser.py b/backend/app/services/layout_parser.py deleted file mode 100644 index c496d3a0..00000000 --- a/backend/app/services/layout_parser.py +++ /dev/null @@ -1,100 +0,0 @@ -import os -from typing import Any, Dict, List - -import fitz # PyMuPDF -import pymupdf4llm -from google import ( - genai, # Since the repo uses Gemini, we'll swap to Gemini 2.5 Flash for vision tasks! -) - -# Initialize Gemini Client -client = genai.Client(api_key=os.environ.get("GEMINI_API_KEY", "dummy_key")) - -class AdvancedPDFParser: - def __init__(self, pdf_path: str): - self.pdf_path = pdf_path - if not os.path.exists(pdf_path): - raise FileNotFoundError(f"PDF file not found at: {pdf_path}") - self.doc = fitz.open(pdf_path) - - def extract_structured_text(self) -> List[Dict[str, Any]]: - """Parses PDF page-by-page preserving markdown layouts & tables.""" - pages_data = [] - try: - md_pages = pymupdf4llm.to_markdown(self.pdf_path, page_chunks=True) - for page in md_pages: - pages_data.append( - { - "page_number": page["metadata"]["page"], - "text": page["text"], - "type": "text_layout", - } - ) - except Exception as e: - print(f"Layout parsing failed, falling back to standard text: {e}") - for page_num in range(len(self.doc)): - page = self.doc.load_page(page_num) - pages_data.append( - { - "page_number": page_num + 1, - "text": page.get_text(), - "type": "fallback_text", - } - ) - return pages_data - - def process_embedded_images(self, page_num: int, page_obj: fitz.Page) -> List[str]: - """Extracts images/charts and uses Gemini Flash to generate dense data descriptions.""" - image_descriptions = [] - image_list = page_obj.get_images(full=True) - - try: - from google import genai - client = genai.Client() - except Exception as e: - print(f"Gemini client init failed, skipping vision: {e}") - return image_descriptions - - for img_index, img in enumerate(image_list): - xref = img[0] - base_image = self.doc.extract_image(xref) - image_bytes = base_image["image"] - - try: - # Use Gemini 2.5 Flash via standard structured part inputs - response = client.models.generate_content( - model="gemini-2.5-flash", - contents=[ - genai.types.Part.from_bytes( - data=image_bytes, mime_type="image/jpeg" - ), - "Analyze this chart/image extracted from a document. Provide a highly detailed summary of its numbers, structural trends, or data contents so it can be effectively used for downstream text retrieval.", - ], - ) - if response.text: - image_descriptions.append(response.text) - except Exception as e: - print(f"Vision processing skipped for page {page_num + 1}: {e}") - continue - - return image_descriptions - - def ingest_document(self) -> List[Dict[str, Any]]: - """Executes the hybrid pipeline generating combined text and image context strings.""" - final_payload = [] - structured_chunks = self.extract_structured_text() - final_payload.extend(structured_chunks) - - for page_num in range(len(self.doc)): - page = self.doc.load_page(page_num) - img_summaries = self.process_embedded_images(page_num, page) - for summary in img_summaries: - final_payload.append( - { - "page_number": page_num + 1, - "text": f"[Visual Data Extraction Summary]: {summary}", - "type": "visual_image_summary", - } - ) - - return final_payload diff --git a/backend/app/tasks.py b/backend/app/tasks.py index d345ef94..d5f6eb0f 100644 --- a/backend/app/tasks.py +++ b/backend/app/tasks.py @@ -1,28 +1,9 @@ """Celery tasks for document processing.""" -import logging -import traceback -from datetime import datetime, timezone - from app.celery_app import celery_app -from app.database import get_db_session -from app.models import Document -from app.services.document_ingestion import ingest_document as _ingest_document - -logger = logging.getLogger(__name__) +from app.services.document_ingestion import ingest_document -@celery_app.task( - bind=True, - name="app.tasks.process_document", - max_retries=3, - default_retry_delay=60, - autoretry_for=(IOError, TimeoutError, ConnectionError, OSError), - retry_backoff=True, - retry_backoff_max=600, - retry_jitter=True, - acks_late=True, - reject_on_worker_lost=True, -) +@celery_app.task(bind=True, name="app.tasks.process_document") def process_document( self, document_id: str, @@ -30,56 +11,12 @@ def process_document( original_name: str, user_id: str, ) -> dict[str, str]: - """Run the RAG ingestion pipeline for a stored document. - - This task is a thin dispatch wrapper around - ``app.services.document_ingestion.ingest_document``, which is the single - source of truth for the ingestion state machine (status, progress, - chunk_count, page_count, summary, URL extraction, knowledge graph, and - vector storage). The task itself only records retry bookkeeping before - delegating; it does not open a second DB session that writes the same - Document row, since ingest_document manages its own SessionLocal() - session end-to-end and commits/rolls back independently. - """ - with get_db_session() as db: - doc = db.query(Document).filter(Document.id == document_id).first() - if doc: - doc.processing_started_at = datetime.now(timezone.utc) - doc.retry_count = (doc.retry_count or 0) + 1 - db.commit() - - logger.info("Dispatching ingestion pipeline for document: %s", original_name) - - try: - _ingest_document( - document_id=document_id, - filepath=filepath, - original_name=original_name, - user_id=user_id, - ) - except Exception as exc: - logger.error( - "Document %s processing failed (attempt %s): %s", - document_id, - self.request.retries + 1, - exc, - ) - with get_db_session() as db: - doc = db.query(Document).filter(Document.id == document_id).first() - if doc and self.request.retries >= (self.max_retries or 3) - 1: - doc.status = "failed" - doc.last_error_traceback = traceback.format_exc()[:2000] - doc.processing_progress = 0 - db.commit() - raise - - with get_db_session() as db: - doc = db.query(Document).filter(Document.id == document_id).first() - final_status = doc.status if doc else "unknown" - - if final_status == "failed": - raise RuntimeError( - f"Ingestion pipeline marked document {document_id} as failed" - ) + """Run the RAG ingestion pipeline for a stored document.""" + ingest_document( + document_id=document_id, + filepath=filepath, + original_name=original_name, + user_id=user_id, + ) + return {"document_id": document_id, "status": "completed"} - return {"document_id": document_id, "status": final_status} \ No newline at end of file diff --git a/backend/migrate_add_extracted_urls.py b/backend/migrate_add_extracted_urls.py deleted file mode 100644 index 8d58730b..00000000 --- a/backend/migrate_add_extracted_urls.py +++ /dev/null @@ -1,32 +0,0 @@ -""" -Migration: add extracted_urls column to documents table. - -Run once: - python migrate_add_extracted_urls.py - -Safe to re-run — skips if the column already exists. -""" -import sys -import os - -sys.path.insert(0, os.path.dirname(__file__)) - -from app.database import engine -from sqlalchemy import text, inspect - -def migrate(): - inspector = inspect(engine) - columns = [col["name"] for col in inspector.get_columns("documents")] - - if "extracted_urls" in columns: - print("Column 'extracted_urls' already exists — skipping.") - return - - with engine.begin() as conn: - conn.execute(text( - "ALTER TABLE documents ADD COLUMN extracted_urls TEXT DEFAULT NULL" - )) - print("Migration complete: added 'extracted_urls' to documents table.") - -if __name__ == "__main__": - migrate() diff --git a/backend/migrate_add_indexes_document.py b/backend/migrate_add_indexes_document.py deleted file mode 100644 index 6a01a9ec..00000000 --- a/backend/migrate_add_indexes_document.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -One-time migration script to add indexes on documents.user_id and documents.filename. -Run this from the 'backend' directory. -""" -import sys -import os - -sys.path.append(os.path.dirname(os.path.abspath(__file__))) - -from app.database import engine -from sqlalchemy import text - -def migrate(): - print("🚀 Starting migration: adding indexes on 'documents' table...") - try: - with engine.connect() as conn: - conn.execute(text( - "CREATE INDEX IF NOT EXISTS ix_documents_user_id ON documents (user_id)" - )) - conn.execute(text( - "CREATE INDEX IF NOT EXISTS ix_documents_filename ON documents (filename)" - )) - conn.commit() - print("✅ Migration successful!") - except Exception as e: - print(f"❌ Migration failed: {e}") - -if __name__ == "__main__": - migrate() \ No newline at end of file diff --git a/backend/migrate_add_processing_columns.py b/backend/migrate_add_processing_columns.py deleted file mode 100644 index 70b7dac3..00000000 --- a/backend/migrate_add_processing_columns.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -Migration: add processing-progress tracking columns and extracted_urls to documents table. - -Adds (idempotent — safe to re-run, skips existing columns): - - processing_progress INTEGER DEFAULT 0 - - processing_stage TEXT DEFAULT 'queued' - - retry_count INTEGER DEFAULT 0 - - last_error_traceback TEXT - - processing_started_at DATETIME - - completed_at DATETIME - - extracted_urls TEXT - -Run once from the backend directory: - python migrate_add_processing_columns.py -""" -import sys -import os - -sys.path.insert(0, os.path.dirname(__file__)) - -from app.database import engine -from sqlalchemy import text, inspect - - -COLUMNS = [ - ("processing_progress", "INTEGER DEFAULT 0"), - ("processing_stage", "TEXT DEFAULT 'queued'"), - ("retry_count", "INTEGER DEFAULT 0"), - ("last_error_traceback", "TEXT DEFAULT NULL"), - ("processing_started_at", "DATETIME DEFAULT NULL"), - ("completed_at", "DATETIME DEFAULT NULL"), - ("extracted_urls", "TEXT DEFAULT NULL"), -] - - -def migrate(): - inspector = inspect(engine) - existing = {col["name"] for col in inspector.get_columns("documents")} - - added = [] - skipped = [] - - with engine.begin() as conn: - for col_name, col_def in COLUMNS: - if col_name in existing: - skipped.append(col_name) - continue - conn.execute(text( - f"ALTER TABLE documents ADD COLUMN {col_name} {col_def}" - )) - added.append(col_name) - - if added: - print(f"✅ Added columns: {added}") - if skipped: - print(f"ℹ️ Already existed (skipped): {skipped}") - if not added: - print("Nothing to do — all columns already present.") - - -if __name__ == "__main__": - migrate() diff --git a/backend/requirements.txt b/backend/requirements.txt index dba3d73f..3f9053c7 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -15,7 +15,6 @@ pyjwt passlib[bcrypt] python-dotenv google-auth -google-auth-oauthlib google-api-python-client APScheduler fastapi-mail @@ -29,12 +28,8 @@ httpx # Document Processing PyMuPDF -pymupdf4llm -google-generativeai -google-genai pdfplumber python-docx -unstructured[pdf] # LangChain & RAG langchain @@ -70,18 +65,9 @@ celery[redis] #brew install libmagic // for OSX python-magic-bin; sys_platform == "win32" # for windows python-magic; sys_platform != "win32" - -# OCR support for scanned/image-based PDFs -pytesseract>=0.3.10 -easyocr>=1.7.1; extra == "easyocr" -Pillow>=10.0.0 - python-docx pypdf reportlab # crawl4ai ddgs -loguru -redis>=4.6.0 -pymupdf4llm -google-genai +google-auth-oauthlib diff --git a/backend/scripts/migrate_sqlite_to_postgres.py b/backend/scripts/migrate_sqlite_to_postgres.py index 1dcc1d05..b7408a51 100644 --- a/backend/scripts/migrate_sqlite_to_postgres.py +++ b/backend/scripts/migrate_sqlite_to_postgres.py @@ -1,66 +1,8 @@ -"""migrate_sqlite_to_postgres.py --------------------------------- -Safely migrate all data from SQLite to PostgreSQL without data loss. - -Supports both the current FastAPI schema (users, api_keys, documents, -chat_messages, shared_messages) and the extended schema that includes -workspaces, workspace_invitations, workspace_members, chat_sessions, -and drive_connections. - -Also supports the older legacy ``instance/users.db`` schema where the -users table is named ``user`` (singular). - -Dependencies ------------- - pip install sqlalchemy psycopg[binary] - # or: pip install sqlalchemy psycopg2-binary - -Tables migrated (FK-safe order) ---------------------------------- - users → api_keys → workspaces → workspace_invitations - → workspace_members → chat_sessions → documents - → chat_messages → drive_connections → shared_messages - -Usage ------ - # Dry-run (reads SQLite, prints counts — no Postgres connection needed): - python migrate_sqlite_to_postgres.py \\ - --sqlite sqlite:///./data/app.db \\ - --dry-run - - # Live migration (Postgres URL via CLI): - python migrate_sqlite_to_postgres.py \\ - --sqlite sqlite:///./data/app.db \\ - --postgres postgresql://user:pass@localhost:5432/mydb - - # Live migration (Postgres URL via environment variable): - export SUPABASE_DB_URL="postgres://user:pass@db.supabase.co:5432/postgres" - python migrate_sqlite_to_postgres.py --sqlite sqlite:///./data/app.db - - # Migrate from the legacy instance/users.db path: - python migrate_sqlite_to_postgres.py \\ - --sqlite-path instance/users.db \\ - --postgres postgresql://user:pass@localhost:5432/mydb - - # Migrate specific tables only: - python migrate_sqlite_to_postgres.py \\ - --sqlite sqlite:///./data/app.db \\ - --postgres postgresql://user:pass@localhost:5432/mydb \\ - --tables users documents chat_messages - - # Wipe Postgres tables before migrating (fresh start): - python migrate_sqlite_to_postgres.py \\ - --sqlite sqlite:///./data/app.db \\ - --postgres postgresql://user:pass@localhost:5432/mydb \\ - --truncate - - # Verbose / debug logging: - python migrate_sqlite_to_postgres.py \\ - --sqlite sqlite:///./data/app.db \\ - --postgres postgresql://user:pass@localhost:5432/mydb \\ - --verbose - -Resolves: #279 +"""Migrate SQLite app data into a Supabase/Postgres database. + +The script supports both the current FastAPI SQLite schema +(`users`, `documents`, `chat_messages`) and the older legacy +`instance/users.db` schema (`user` only). """ from __future__ import annotations @@ -69,70 +11,115 @@ import os import sys import uuid +from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path -from typing import Any, Dict, List, Optional - -from sqlalchemy import create_engine, inspect, text -from sqlalchemy.exc import OperationalError - -# --------------------------------------------------------------------------- -# Logging -# --------------------------------------------------------------------------- -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s %(levelname)-8s %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", +from typing import Any + +from sqlalchemy import ( + Boolean, + Column, + DateTime, + ForeignKey, + Integer, + MetaData, + String, + Table, + Text, + create_engine, + inspect, + select, ) -logger = logging.getLogger("migration") +from sqlalchemy.engine import Engine +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session, sessionmaker + +LOGGER = logging.getLogger("sqlite_to_postgres") + + +def generate_uuid() -> str: + return str(uuid.uuid4()) + -# --------------------------------------------------------------------------- -# Migration order — respects FK dependencies -# --------------------------------------------------------------------------- -TABLE_ORDER = [ +metadata = MetaData() + +users = Table( "users", + metadata, + Column("id", String, primary_key=True, default=generate_uuid), + Column("username", String(80), unique=True, nullable=False, index=True), + Column("email", String(120), unique=True, nullable=False, index=True), + Column("hashed_password", String(255), nullable=False), + Column("is_admin", Boolean, default=False), + Column("created_at", DateTime, default=lambda: datetime.now(timezone.utc)), + Column("last_login", DateTime, nullable=True, index=True), + Column("hf_token", String(255), nullable=True), +) + +api_keys = Table( "api_keys", - "workspaces", - "workspace_invitations", - "workspace_members", - "chat_sessions", + metadata, + Column("id", String, primary_key=True, default=generate_uuid), + Column("user_id", String, ForeignKey("users.id"), nullable=False, index=True), + Column("key_prefix", String(10), nullable=False), + Column("hashed_key", String(255), nullable=False, unique=True, index=True), + Column("created_at", DateTime, default=lambda: datetime.now(timezone.utc)), + Column("last_used", DateTime, nullable=True), +) + +documents = Table( "documents", + metadata, + Column("id", String, primary_key=True, default=generate_uuid), + Column("user_id", String, ForeignKey("users.id"), nullable=False, index=True), + Column("filename", String(255), nullable=False), + Column("original_name", String(255), nullable=False), + Column("file_size", Integer, default=0), + Column("page_count", Integer, default=0), + Column("chunk_count", Integer, default=0), + Column("status", String(20), default="pending"), + Column("error_message", Text, nullable=True), + Column("uploaded_at", DateTime, default=lambda: datetime.now(timezone.utc)), + Column("summary", Text, nullable=True), +) + +chat_messages = Table( "chat_messages", - "drive_connections", + metadata, + Column("id", String, primary_key=True, default=generate_uuid), + Column("user_id", String, ForeignKey("users.id"), nullable=False, index=True), + Column("document_id", String, ForeignKey("documents.id"), nullable=True, index=True), + Column("role", String(20), nullable=False), + Column("content", Text, nullable=False), + Column("sources_json", Text, nullable=True), + Column("created_at", DateTime, default=lambda: datetime.now(timezone.utc)), +) + +shared_messages = Table( "shared_messages", -] + metadata, + Column("id", String, primary_key=True, default=generate_uuid), + Column("message_id", String, ForeignKey("chat_messages.id"), nullable=False, unique=True, index=True), + Column("created_at", DateTime, default=lambda: datetime.now(timezone.utc)), +) -# Columns that hold UUID / GUID values and need string normalisation -_UUID_COLUMNS = { - "id", "user_id", "document_id", "session_id", "message_id", - "workspace_id", "inviter_id", "created_by", -} -# Boolean columns stored as 0/1 integers in SQLite -_BOOL_COLUMNS = { - "is_admin", "is_verified", "is_active", "is_deleted", "enabled", -} +@dataclass +class MigrationStats: + inserted: dict[str, int] = field(default_factory=dict) + reused: dict[str, int] = field(default_factory=dict) + skipped: dict[str, int] = field(default_factory=dict) + def add(self, table_name: str, action: str) -> None: + getattr(self, action)[table_name] = getattr(self, action).get(table_name, 0) + 1 -# --------------------------------------------------------------------------- -# URL helpers -# --------------------------------------------------------------------------- def normalize_postgres_url(url: str) -> str: - """ - Normalise common Postgres URL forms to use the psycopg (v3) driver. - - Supabase and many hosting providers hand out ``postgres://`` or plain - ``postgresql://`` connection strings. SQLAlchemy requires a driver - specifier such as ``postgresql+psycopg://`` for psycopg v3, or - ``postgresql+psycopg2://`` for psycopg2. We prefer psycopg v3 when - no driver is already specified. - """ + """Prefer psycopg v3 when callers pass Supabase's common URL forms.""" if url.startswith("postgres://"): - url = "postgresql+psycopg://" + url[len("postgres://"):] - elif url.startswith("postgresql://"): - url = "postgresql+psycopg://" + url[len("postgresql://"):] - # Already has a driver specifier (e.g. postgresql+psycopg2://) — leave it. + return "postgresql+psycopg://" + url.removeprefix("postgres://") + if url.startswith("postgresql://"): + return "postgresql+psycopg://" + url.removeprefix("postgresql://") return url @@ -140,433 +127,398 @@ def sqlite_url_from_path(path: str) -> str: return f"sqlite:///{Path(path).resolve().as_posix()}" -# --------------------------------------------------------------------------- -# Engine factory -# --------------------------------------------------------------------------- +def make_engine(url: str) -> Engine: + return create_engine(url, future=True) -def _make_engine(url: str, label: str): - """Create a SQLAlchemy engine with sensible defaults.""" - is_sqlite = url.startswith("sqlite") - kwargs: Dict[str, Any] = {"echo": False, "future": True} - if is_sqlite: - kwargs["connect_args"] = {"check_same_thread": False} - else: - kwargs.update(pool_pre_ping=True, pool_size=5, max_overflow=10) - logger.info("Connecting to %s: %s", label, url) - return create_engine(url, **kwargs) +def make_session(engine: Engine) -> Session: + return sessionmaker(bind=engine, autocommit=False, autoflush=False, future=True)() -# --------------------------------------------------------------------------- -# Row coercion -# --------------------------------------------------------------------------- -def _normalise_uuid(value: Any) -> Optional[str]: - """Return a UUID as a plain lowercase string, or None.""" - if value is None: +def reflected_table(engine: Engine, table_name: str) -> Table | None: + if not inspect(engine).has_table(table_name): return None - if isinstance(value, uuid.UUID): - return str(value) - try: - return str(uuid.UUID(str(value))) - except (ValueError, AttributeError): - return str(value) - - -def _coerce_row(row: Dict[str, Any]) -> Dict[str, Any]: - """ - Coerce a SQLite row dict so it is safe to insert into Postgres: - - * UUID columns → normalised lowercase str - * Boolean integers → Python bool (SQLite stores True/False as 1/0) - * Naive datetimes → UTC-aware datetimes - """ - result: Dict[str, Any] = {} - for col, val in row.items(): - if col in _UUID_COLUMNS: - result[col] = _normalise_uuid(val) - elif col in _BOOL_COLUMNS: - result[col] = bool(val) if val is not None else None - elif isinstance(val, datetime) and val.tzinfo is None: - result[col] = val.replace(tzinfo=timezone.utc) - else: - result[col] = val - return result - + reflected = MetaData() + return Table(table_name, reflected, autoload_with=engine) -# --------------------------------------------------------------------------- -# Table helpers -# --------------------------------------------------------------------------- -def _table_exists(inspector, table: str) -> bool: - return table in inspector.get_table_names() +def fetch_rows(session: Session, table: Table) -> list[dict[str, Any]]: + stmt = select(table) + if "id" in table.c: + stmt = stmt.order_by(table.c.id) + return [dict(row) for row in session.execute(stmt).mappings().all()] -def _resolve_users_table(src_inspector) -> Optional[str]: - """ - Return the name of the users table in the source database. - - Supports both the current schema (``users``) and the legacy schema - (``user``, singular) used by older ``instance/users.db`` databases. - Returns None if neither is found. - """ - if _table_exists(src_inspector, "users"): - return "users" - if _table_exists(src_inspector, "user"): - logger.info("Legacy 'user' table detected — will migrate as 'users'.") - return "user" - return None - - -# --------------------------------------------------------------------------- -# Per-table migration -# --------------------------------------------------------------------------- - -def _dry_run_table(table: str, src_conn) -> Dict[str, int]: - """Read SQLite row counts and log them — no Postgres connection used.""" - src_inspector = inspect(src_conn) - if not _table_exists(src_inspector, table): - logger.warning(" [%s] not found in SQLite — skipping", table) - return {"would_migrate": 0, "skipped": 0} - - count = src_conn.execute(text(f"SELECT COUNT(*) FROM \"{table}\"")).scalar() or 0 - logger.info(" [%s] %d rows found (dry-run — nothing written)", table, count) - return {"would_migrate": count, "skipped": 0} - - -def _live_migrate_table( - table: str, - src_table_name: str, - src_conn, - dst_conn, - truncate: bool, - batch_size: int, -) -> Dict[str, int]: - """ - Migrate one table from SQLite → Postgres. - - ``table`` is the canonical destination table name (always the current - schema name, e.g. ``"users"``). ``src_table_name`` may differ for - legacy sources (e.g. ``"user"``). - """ - stats = {"inserted": 0, "skipped": 0} - - src_inspector = inspect(src_conn) - dst_inspector = inspect(dst_conn) - - if not _table_exists(src_inspector, src_table_name): - logger.warning(" [%s] not found in SQLite — skipping", table) - return stats +def existing_id(session: Session, table: Table, source_id: str | None) -> str | None: + if not source_id: + return None + return session.execute(select(table.c.id).where(table.c.id == source_id)).scalar_one_or_none() - if not _table_exists(dst_inspector, table): - logger.warning( - " [%s] not found in Postgres — skipping. " - "Run your init_postgres.sql first to create the schema.", - table, - ) - return stats - # Fetch all rows from source - rows_result = src_conn.execute(text(f"SELECT * FROM \"{src_table_name}\"")) - column_names: List[str] = list(rows_result.keys()) - raw_rows = rows_result.fetchall() +def available_id(session: Session, table: Table, source_id: Any) -> str: + candidate = str(source_id) if source_id is not None else generate_uuid() + if existing_id(session, table, candidate) is None: + return candidate - if not raw_rows: - logger.info(" [%s] 0 rows — nothing to migrate", table) - return stats + while True: + candidate = generate_uuid() + if existing_id(session, table, candidate) is None: + return candidate - logger.info(" [%s] %d rows to migrate", table, len(raw_rows)) - if truncate: - logger.info(" [%s] truncating Postgres table (CASCADE)", table) - dst_conn.execute(text(f'TRUNCATE TABLE "{table}" RESTART IDENTITY CASCADE')) +def first_existing_user(session: Session, row: dict[str, Any]) -> str | None: + email = row.get("email") + username = row.get("username") + if email: + match = session.execute(select(users.c.id).where(users.c.email == email)).scalar_one_or_none() + if match: + return match + if username: + return session.execute(select(users.c.id).where(users.c.username == username)).scalar_one_or_none() + return None - # Build idempotent INSERT — skips duplicates silently - col_list = ", ".join(f'"{c}"' for c in column_names) - placeholders = ", ".join(f":{c}" for c in column_names) - insert_sql = text( - f'INSERT INTO "{table}" ({col_list}) ' - f"VALUES ({placeholders}) " - f"ON CONFLICT DO NOTHING" - ) - batch: List[Dict[str, Any]] = [] - for raw in raw_rows: - row_dict = dict(zip(column_names, raw)) - batch.append(_coerce_row(row_dict)) +def copy_users( + source_session: Session, + target_session: Session, + source_table: Table, + stats: MigrationStats, +) -> dict[str, str]: + id_map: dict[str, str] = {} + now = datetime.now(timezone.utc) + + for row in fetch_rows(source_session, source_table): + old_id = str(row.get("id")) + existing = existing_id(target_session, users, old_id) or first_existing_user(target_session, row) + if existing: + id_map[old_id] = existing + stats.add("users", "reused") + continue + + is_legacy = source_table.name == "user" + new_id = available_id(target_session, users, None if is_legacy else old_id) + user_values = { + "id": new_id, + "username": row["username"], + "email": row["email"], + "hashed_password": row.get("hashed_password") or row.get("password") or "", + "is_admin": bool(row.get("is_admin") or False), + "created_at": row.get("created_at") or now, + "last_login": row.get("last_login"), + "hf_token": row.get("hf_token"), + } + target_session.execute(users.insert().values(**user_values)) + id_map[old_id] = new_id + stats.add("users", "inserted") + + return id_map + + +def copy_api_keys( + source_session: Session, + target_session: Session, + source_table: Table | None, + user_id_map: dict[str, str], + stats: MigrationStats, +) -> dict[str, str]: + id_map: dict[str, str] = {} + if source_table is None: + return id_map + + for row in fetch_rows(source_session, source_table): + old_id = str(row.get("id")) + new_user_id = user_id_map.get(str(row.get("user_id"))) + if not new_user_id: + stats.add("api_keys", "skipped") + continue + + existing = ( + existing_id(target_session, api_keys, old_id) + or target_session.execute( + select(api_keys.c.id).where(api_keys.c.hashed_key == row.get("hashed_key")) + ).scalar_one_or_none() + ) + if existing: + id_map[old_id] = existing + stats.add("api_keys", "reused") + continue + + new_id = available_id(target_session, api_keys, old_id) + target_session.execute( + api_keys.insert().values( + id=new_id, + user_id=new_user_id, + key_prefix=row["key_prefix"], + hashed_key=row["hashed_key"], + created_at=row.get("created_at") or datetime.now(timezone.utc), + last_used=row.get("last_used"), + ) + ) + id_map[old_id] = new_id + stats.add("api_keys", "inserted") + + return id_map + + +def copy_documents( + source_session: Session, + target_session: Session, + source_table: Table | None, + user_id_map: dict[str, str], + stats: MigrationStats, +) -> dict[str, str]: + id_map: dict[str, str] = {} + if source_table is None: + return id_map + + for row in fetch_rows(source_session, source_table): + old_id = str(row.get("id")) + new_user_id = user_id_map.get(str(row.get("user_id"))) + if not new_user_id: + stats.add("documents", "skipped") + continue + + existing = existing_id(target_session, documents, old_id) + if existing: + id_map[old_id] = existing + stats.add("documents", "reused") + continue + + new_id = available_id(target_session, documents, old_id) + target_session.execute( + documents.insert().values( + id=new_id, + user_id=new_user_id, + filename=row["filename"], + original_name=row["original_name"], + file_size=row.get("file_size") or 0, + page_count=row.get("page_count") or 0, + chunk_count=row.get("chunk_count") or 0, + status=row.get("status") or "pending", + error_message=row.get("error_message"), + uploaded_at=row.get("uploaded_at") or datetime.now(timezone.utc), + summary=row.get("summary"), + ) + ) + id_map[old_id] = new_id + stats.add("documents", "inserted") + + return id_map + + +def copy_chat_messages( + source_session: Session, + target_session: Session, + source_table: Table | None, + user_id_map: dict[str, str], + document_id_map: dict[str, str], + stats: MigrationStats, +) -> dict[str, str]: + id_map: dict[str, str] = {} + if source_table is None: + return id_map + + for row in fetch_rows(source_session, source_table): + old_id = str(row.get("id")) + new_user_id = user_id_map.get(str(row.get("user_id"))) + old_document_id = row.get("document_id") + new_document_id = document_id_map.get(str(old_document_id)) if old_document_id else None + if not new_user_id or (old_document_id and not new_document_id): + stats.add("chat_messages", "skipped") + continue + + existing = existing_id(target_session, chat_messages, old_id) + if existing: + id_map[old_id] = existing + stats.add("chat_messages", "reused") + continue + + new_id = available_id(target_session, chat_messages, old_id) + target_session.execute( + chat_messages.insert().values( + id=new_id, + user_id=new_user_id, + document_id=new_document_id, + role=row["role"], + content=row["content"], + sources_json=row.get("sources_json"), + created_at=row.get("created_at") or datetime.now(timezone.utc), + ) + ) + id_map[old_id] = new_id + stats.add("chat_messages", "inserted") + + return id_map + + +def copy_shared_messages( + source_session: Session, + target_session: Session, + source_table: Table | None, + message_id_map: dict[str, str], + stats: MigrationStats, +) -> None: + if source_table is None: + return + + for row in fetch_rows(source_session, source_table): + old_id = str(row.get("id")) + new_message_id = message_id_map.get(str(row.get("message_id"))) + if not new_message_id: + stats.add("shared_messages", "skipped") + continue + + existing = ( + existing_id(target_session, shared_messages, old_id) + or target_session.execute( + select(shared_messages.c.id).where(shared_messages.c.message_id == new_message_id) + ).scalar_one_or_none() + ) + if existing: + stats.add("shared_messages", "reused") + continue + + target_session.execute( + shared_messages.insert().values( + id=available_id(target_session, shared_messages, old_id), + message_id=new_message_id, + created_at=row.get("created_at") or datetime.now(timezone.utc), + ) + ) + stats.add("shared_messages", "inserted") - if len(batch) >= batch_size: - result = dst_conn.execute(insert_sql, batch) - stats["inserted"] += result.rowcount if result.rowcount >= 0 else len(batch) - batch = [] - if batch: - result = dst_conn.execute(insert_sql, batch) - stats["inserted"] += result.rowcount if result.rowcount >= 0 else len(batch) +def migrate( + sqlite_url: str, + postgres_url: str, + create_tables: bool, + dry_run: bool, +) -> MigrationStats: + source_engine = make_engine(sqlite_url) + target_engine = make_engine(normalize_postgres_url(postgres_url)) - logger.info(" [%s] inserted %d rows", table, stats["inserted"]) - return stats + if create_tables: + metadata.create_all(target_engine) + source_session = make_session(source_engine) + target_session = make_session(target_engine) + stats = MigrationStats() -# --------------------------------------------------------------------------- -# Orchestration -# --------------------------------------------------------------------------- + try: + current_users = reflected_table(source_engine, "users") + legacy_users = reflected_table(source_engine, "user") + source_users = current_users if current_users is not None else legacy_users + if source_users is None: + raise RuntimeError("No users table found. Expected 'users' or legacy 'user'.") + + user_id_map = copy_users(source_session, target_session, source_users, stats) + copy_api_keys(source_session, target_session, reflected_table(source_engine, "api_keys"), user_id_map, stats) + document_id_map = copy_documents( + source_session, + target_session, + reflected_table(source_engine, "documents"), + user_id_map, + stats, + ) + message_id_map = copy_chat_messages( + source_session, + target_session, + reflected_table(source_engine, "chat_messages"), + user_id_map, + document_id_map, + stats, + ) + copy_shared_messages( + source_session, + target_session, + reflected_table(source_engine, "shared_messages"), + message_id_map, + stats, + ) -def run_migration( - sqlite_url: str, - postgres_url: Optional[str], - dry_run: bool, - truncate: bool, - tables: Optional[List[str]], - batch_size: int, -) -> bool: - """ - Orchestrate the full migration. Returns True on success. - - In dry-run mode, no Postgres connection is opened; only SQLite row - counts are reported. - """ - # Build the ordered table list - target = set(tables) if tables else set(TABLE_ORDER) - ordered = [t for t in TABLE_ORDER if t in target] - # Append any user-supplied extras that aren't in TABLE_ORDER - for t in (tables or []): - if t not in ordered: - logger.warning( - "Table '%s' is not in the known FK-safe order. " - "It will be migrated last — ensure FK dependencies are satisfied.", - t, - ) - ordered.append(t) - - mode = "DRY RUN" if dry_run else "LIVE" - logger.info("=" * 60) - logger.info("Migration mode : %s", mode) - logger.info("Source : %s", sqlite_url) - if not dry_run: - logger.info("Destination : %s", postgres_url) - logger.info("Tables : %s", ", ".join(ordered)) - logger.info("Truncate first : %s", truncate and not dry_run) - logger.info("Batch size : %d", batch_size) - logger.info("=" * 60) - - src_engine = _make_engine(sqlite_url, "SQLite") - - if dry_run: - overall_stats: Dict[str, Dict[str, int]] = {} - with src_engine.connect() as src_conn: - src_inspector = inspect(src_conn) - users_src = _resolve_users_table(src_inspector) - - for table in ordered: - logger.info("Checking table : %s", table) - # For the legacy 'user' → 'users' mapping - src_table_name = users_src if table == "users" and users_src else table - if src_table_name is None: - logger.error("No users/user table found in SQLite.") - return False - overall_stats[table] = _dry_run_table(src_table_name, src_conn) - else: - if not postgres_url: - logger.error( - "A Postgres URL is required for a live migration. " - "Pass --postgres or set SUPABASE_DB_URL / DATABASE_URL." - ) - return False - - pg_url = normalize_postgres_url(postgres_url) - dst_engine = _make_engine(pg_url, "PostgreSQL") - - # Verify Postgres connectivity before touching data - try: - with dst_engine.connect() as probe: - probe.execute(text("SELECT 1")) - except OperationalError as exc: - logger.error("Cannot connect to Postgres: %s", exc) - return False - - overall_stats = {} - # Single Postgres transaction — full rollback on any error - with src_engine.connect() as src_conn, dst_engine.begin() as dst_conn: - src_inspector = inspect(src_conn) - users_src = _resolve_users_table(src_inspector) - - for table in ordered: - logger.info("Migrating table: %s", table) - src_table_name = users_src if table == "users" and users_src else table - if table == "users" and src_table_name is None: - logger.error("No users/user table found in SQLite — aborting.") - raise RuntimeError("Missing users table in source database.") - try: - overall_stats[table] = _live_migrate_table( - table=table, - src_table_name=src_table_name or table, - src_conn=src_conn, - dst_conn=dst_conn, - truncate=truncate, - batch_size=batch_size, - ) - except Exception as exc: - logger.error( - " [%s] FAILED — rolling back all changes: %s", - table, - exc, - exc_info=True, - ) - raise # triggers dst_engine.begin() rollback - - # Print summary - logger.info("=" * 60) - logger.info("Summary") - logger.info("-" * 60) - total_migrated = 0 - for table, stats in overall_stats.items(): if dry_run: - logger.info( - " %-30s would_migrate=%-6d skipped=%d", - table, - stats.get("would_migrate", 0), - stats.get("skipped", 0), - ) - total_migrated += stats.get("would_migrate", 0) + target_session.rollback() + LOGGER.info("Dry run complete; rolled back target transaction.") else: - logger.info( - " %-30s inserted=%-6d skipped=%d", - table, - stats.get("inserted", 0), - stats.get("skipped", 0), - ) - total_migrated += stats.get("inserted", 0) - - logger.info("-" * 60) - if dry_run: - logger.info(" TOTAL would_migrate=%d", total_migrated) - logger.info("Dry run complete — no data was written to Postgres.") - else: - logger.info(" TOTAL inserted=%d", total_migrated) - logger.info("Migration complete.") - logger.info("=" * 60) - return True + target_session.commit() + LOGGER.info("Migration committed.") + return stats + except IntegrityError: + target_session.rollback() + LOGGER.exception("Migration failed because the target database rejected a row.") + raise + except Exception: + target_session.rollback() + LOGGER.exception("Migration failed; rolled back target transaction.") + raise + finally: + source_session.close() + target_session.close() + source_engine.dispose() + target_engine.dispose() -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Migrate data from SQLite to PostgreSQL.", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=__doc__, - ) - - src_group = parser.add_mutually_exclusive_group() - src_group.add_argument( - "--sqlite", - metavar="URL", - help="SQLAlchemy URL for the SQLite source, e.g. sqlite:///./data/app.db", - ) - src_group.add_argument( + parser = argparse.ArgumentParser(description="Migrate SQLite users/documents/chat history to Supabase Postgres.") + parser.add_argument( "--sqlite-path", - metavar="PATH", default="instance/users.db", - help="Path to the SQLite file (alternative to --sqlite). " - "Defaults to instance/users.db.", + help="Path to the SQLite database file. Defaults to instance/users.db.", ) - parser.add_argument( - "--postgres", - metavar="URL", - default=( - os.getenv("SUPABASE_DB_URL") - or os.getenv("POSTGRES_DATABASE_URL") - or os.getenv("DATABASE_URL") - ), - help="Postgres destination URL (postgresql:// or postgres://). " - "Also read from SUPABASE_DB_URL, POSTGRES_DATABASE_URL, or DATABASE_URL. " - "Not required for --dry-run.", + "--sqlite-url", + help="Full SQLite SQLAlchemy URL. Overrides --sqlite-path.", ) parser.add_argument( - "--dry-run", - action="store_true", - default=False, - help="Read from SQLite and report row counts — no Postgres connection made.", + "--postgres-url", + default=os.getenv("SUPABASE_DB_URL") or os.getenv("POSTGRES_DATABASE_URL") or os.getenv("DATABASE_URL"), + help="Supabase/Postgres SQLAlchemy URL. Also read from SUPABASE_DB_URL, POSTGRES_DATABASE_URL, or DATABASE_URL.", ) parser.add_argument( - "--truncate", + "--no-create-tables", action="store_true", - default=False, - help="TRUNCATE each Postgres table (CASCADE) before inserting. " - "Useful for a clean re-migration. Ignored during --dry-run.", - ) - parser.add_argument( - "--tables", - nargs="+", - metavar="TABLE", - help="Migrate only these tables (space-separated). " - "Defaults to all tables in FK-safe order.", + help="Do not create missing target tables before migrating.", ) parser.add_argument( - "--batch-size", - type=int, - default=500, - metavar="N", - help="Number of rows per INSERT batch (default: 500).", - ) - parser.add_argument( - "--verbose", + "--dry-run", action="store_true", - help="Enable DEBUG logging.", + help="Run the migration and roll back the target transaction.", ) + parser.add_argument("--verbose", action="store_true", help="Enable debug logging.") return parser.parse_args() -def main() -> None: +def main() -> int: args = parse_args() + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(levelname)s %(message)s", + ) - if args.verbose: - logging.getLogger().setLevel(logging.DEBUG) - logger.setLevel(logging.DEBUG) - - # Resolve SQLite URL - if args.sqlite: - sqlite_url = args.sqlite - if not sqlite_url.startswith("sqlite"): - logger.error("--sqlite must be a sqlite:/// URL, got: %s", sqlite_url) - sys.exit(1) - else: - sqlite_url = sqlite_url_from_path(args.sqlite_path) - - # Postgres URL is optional for dry-run only - postgres_url: Optional[str] = args.postgres - if not args.dry_run: - if not postgres_url: - logger.error( - "Provide a Postgres URL via --postgres, SUPABASE_DB_URL, " - "POSTGRES_DATABASE_URL, or DATABASE_URL." - ) - sys.exit(2) - if postgres_url.startswith("sqlite"): - logger.error( - "--postgres must be a Postgres URL, not a SQLite URL: %s", - postgres_url, - ) - sys.exit(2) + postgres_url = args.postgres_url + if not postgres_url or postgres_url.startswith("sqlite"): + LOGGER.error("Provide a Supabase/Postgres URL with --postgres-url or SUPABASE_DB_URL.") + return 2 + + sqlite_url = args.sqlite_url or sqlite_url_from_path(args.sqlite_path) + stats = migrate( + sqlite_url=sqlite_url, + postgres_url=postgres_url, + create_tables=not args.no_create_tables, + dry_run=args.dry_run, + ) - try: - success = run_migration( - sqlite_url=sqlite_url, - postgres_url=postgres_url, - dry_run=args.dry_run, - truncate=args.truncate, - tables=args.tables, - batch_size=args.batch_size, + for table_name in sorted(set(stats.inserted) | set(stats.reused) | set(stats.skipped)): + LOGGER.info( + "%s: inserted=%s reused=%s skipped=%s", + table_name, + stats.inserted.get(table_name, 0), + stats.reused.get(table_name, 0), + stats.skipped.get(table_name, 0), ) - except Exception as exc: - logger.error("Migration failed: %s", exc) - sys.exit(1) - - sys.exit(0 if success else 1) + return 0 if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/backend/test_uploads/0229378a-9e5e-4661-a6a1-5c7be098a8b9/0ae026101e3745dbbe307b8f01c9bcf6.txt b/backend/test_uploads/0229378a-9e5e-4661-a6a1-5c7be098a8b9/0ae026101e3745dbbe307b8f01c9bcf6.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/0229378a-9e5e-4661-a6a1-5c7be098a8b9/0ae026101e3745dbbe307b8f01c9bcf6.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/0229378a-9e5e-4661-a6a1-5c7be098a8b9/7007e396738e449ca5cfdfc0c28d585a.txt b/backend/test_uploads/0229378a-9e5e-4661-a6a1-5c7be098a8b9/7007e396738e449ca5cfdfc0c28d585a.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/0229378a-9e5e-4661-a6a1-5c7be098a8b9/7007e396738e449ca5cfdfc0c28d585a.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/0229378a-9e5e-4661-a6a1-5c7be098a8b9/deed06f96f9e4010935ba50de68856fd.txt b/backend/test_uploads/0229378a-9e5e-4661-a6a1-5c7be098a8b9/deed06f96f9e4010935ba50de68856fd.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/0229378a-9e5e-4661-a6a1-5c7be098a8b9/deed06f96f9e4010935ba50de68856fd.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/02df53d0-42b2-462e-9084-d5c93cc413f0/5c569a9c03d044e7a519ed1d437b1447.txt b/backend/test_uploads/02df53d0-42b2-462e-9084-d5c93cc413f0/5c569a9c03d044e7a519ed1d437b1447.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/02df53d0-42b2-462e-9084-d5c93cc413f0/5c569a9c03d044e7a519ed1d437b1447.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/077c2d3a-bd42-4fb5-8674-5c523df4cf7e/9727844b0724434ab563de7a2df64124.txt b/backend/test_uploads/077c2d3a-bd42-4fb5-8674-5c523df4cf7e/9727844b0724434ab563de7a2df64124.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/077c2d3a-bd42-4fb5-8674-5c523df4cf7e/9727844b0724434ab563de7a2df64124.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/163f69df-9017-4a40-8964-0f069258d8c3/9f42201f3d904f4e87d3b5ac02b5b9d6.txt b/backend/test_uploads/163f69df-9017-4a40-8964-0f069258d8c3/9f42201f3d904f4e87d3b5ac02b5b9d6.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/163f69df-9017-4a40-8964-0f069258d8c3/9f42201f3d904f4e87d3b5ac02b5b9d6.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/321d8b9a-fe39-46ee-9299-2f44a7a82a1c/1603f827c96949cfaff7921e3f369f64.txt b/backend/test_uploads/321d8b9a-fe39-46ee-9299-2f44a7a82a1c/1603f827c96949cfaff7921e3f369f64.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/321d8b9a-fe39-46ee-9299-2f44a7a82a1c/1603f827c96949cfaff7921e3f369f64.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/37737036-5479-43f5-b9e0-7d1107844e87/458189ec8eb64baa9dabe1b6b229cada.txt b/backend/test_uploads/37737036-5479-43f5-b9e0-7d1107844e87/458189ec8eb64baa9dabe1b6b229cada.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/37737036-5479-43f5-b9e0-7d1107844e87/458189ec8eb64baa9dabe1b6b229cada.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/3d804f9a-34d1-4703-84d5-bca33917546c/7d7c4532c5f6493f8d075e711acf6a6b.txt b/backend/test_uploads/3d804f9a-34d1-4703-84d5-bca33917546c/7d7c4532c5f6493f8d075e711acf6a6b.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/3d804f9a-34d1-4703-84d5-bca33917546c/7d7c4532c5f6493f8d075e711acf6a6b.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/5b860822-179d-4765-bf47-f31f6c04f6c0/b894992bd91b4b79843b43f07aa8d58d.txt b/backend/test_uploads/5b860822-179d-4765-bf47-f31f6c04f6c0/b894992bd91b4b79843b43f07aa8d58d.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/5b860822-179d-4765-bf47-f31f6c04f6c0/b894992bd91b4b79843b43f07aa8d58d.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/71397b44-6209-4223-9277-d98eb41ff164/7647003dbefd40adac19b9145fc501e4.txt b/backend/test_uploads/71397b44-6209-4223-9277-d98eb41ff164/7647003dbefd40adac19b9145fc501e4.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/71397b44-6209-4223-9277-d98eb41ff164/7647003dbefd40adac19b9145fc501e4.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/7348dfae-e3e2-4b87-8169-439b9c5cbe69/857c9de7d2104b79a7b0fb2caecefabc.txt b/backend/test_uploads/7348dfae-e3e2-4b87-8169-439b9c5cbe69/857c9de7d2104b79a7b0fb2caecefabc.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/7348dfae-e3e2-4b87-8169-439b9c5cbe69/857c9de7d2104b79a7b0fb2caecefabc.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/7348dfae-e3e2-4b87-8169-439b9c5cbe69/9fbf4bd51bfb46138c1d96c6014ebf8f.txt b/backend/test_uploads/7348dfae-e3e2-4b87-8169-439b9c5cbe69/9fbf4bd51bfb46138c1d96c6014ebf8f.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/7348dfae-e3e2-4b87-8169-439b9c5cbe69/9fbf4bd51bfb46138c1d96c6014ebf8f.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/7348dfae-e3e2-4b87-8169-439b9c5cbe69/f17ec52a3b814e40b21e7e1e0c357069.txt b/backend/test_uploads/7348dfae-e3e2-4b87-8169-439b9c5cbe69/f17ec52a3b814e40b21e7e1e0c357069.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/7348dfae-e3e2-4b87-8169-439b9c5cbe69/f17ec52a3b814e40b21e7e1e0c357069.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/7cded8ed-2258-4888-b5c7-e650178da057/6e016a5c837c44968c06c84c92bba0d2.txt b/backend/test_uploads/7cded8ed-2258-4888-b5c7-e650178da057/6e016a5c837c44968c06c84c92bba0d2.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/7cded8ed-2258-4888-b5c7-e650178da057/6e016a5c837c44968c06c84c92bba0d2.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/8191dbd4-fea0-4869-9623-5d9945d9555d/1b1cc690509b4f9c9ce5454de0df597f.txt b/backend/test_uploads/8191dbd4-fea0-4869-9623-5d9945d9555d/1b1cc690509b4f9c9ce5454de0df597f.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/8191dbd4-fea0-4869-9623-5d9945d9555d/1b1cc690509b4f9c9ce5454de0df597f.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/92679fb1-3961-4409-a25a-002dd754e3f3/c7f346441fb5439da0ea51870e130496.txt b/backend/test_uploads/92679fb1-3961-4409-a25a-002dd754e3f3/c7f346441fb5439da0ea51870e130496.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/92679fb1-3961-4409-a25a-002dd754e3f3/c7f346441fb5439da0ea51870e130496.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/9a50ad10-ef1b-4309-ba4e-59fd84db72e8/1a35632cb7a74f06a39da86644f9f8ae.txt b/backend/test_uploads/9a50ad10-ef1b-4309-ba4e-59fd84db72e8/1a35632cb7a74f06a39da86644f9f8ae.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/9a50ad10-ef1b-4309-ba4e-59fd84db72e8/1a35632cb7a74f06a39da86644f9f8ae.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/9a50ad10-ef1b-4309-ba4e-59fd84db72e8/e3c4af57da76402f863d3b55999aa115.txt b/backend/test_uploads/9a50ad10-ef1b-4309-ba4e-59fd84db72e8/e3c4af57da76402f863d3b55999aa115.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/9a50ad10-ef1b-4309-ba4e-59fd84db72e8/e3c4af57da76402f863d3b55999aa115.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/9a50ad10-ef1b-4309-ba4e-59fd84db72e8/fb2616027e3649eb8466b3d4cc950b1a.txt b/backend/test_uploads/9a50ad10-ef1b-4309-ba4e-59fd84db72e8/fb2616027e3649eb8466b3d4cc950b1a.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/9a50ad10-ef1b-4309-ba4e-59fd84db72e8/fb2616027e3649eb8466b3d4cc950b1a.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/aa12cc9a-e946-4c63-bf81-3c383cd86557/7c2f00a5053346f1aa52d19be75eb9b5.txt b/backend/test_uploads/aa12cc9a-e946-4c63-bf81-3c383cd86557/7c2f00a5053346f1aa52d19be75eb9b5.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/aa12cc9a-e946-4c63-bf81-3c383cd86557/7c2f00a5053346f1aa52d19be75eb9b5.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/abeb0768-1f46-495f-8eab-2305966e6600/7d990e3ec5934ae0945271356f7f023d.txt b/backend/test_uploads/abeb0768-1f46-495f-8eab-2305966e6600/7d990e3ec5934ae0945271356f7f023d.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/abeb0768-1f46-495f-8eab-2305966e6600/7d990e3ec5934ae0945271356f7f023d.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/ac763668-e6c6-49e0-903a-3dbe56f57b83/1df6f89ddc4845c6b3e73d8886a0a4de.txt b/backend/test_uploads/ac763668-e6c6-49e0-903a-3dbe56f57b83/1df6f89ddc4845c6b3e73d8886a0a4de.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/ac763668-e6c6-49e0-903a-3dbe56f57b83/1df6f89ddc4845c6b3e73d8886a0a4de.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/cf8ec409-c477-46de-8213-9c3b0236ba3f/381c3ddc8c31479e8422e485805327cd.txt b/backend/test_uploads/cf8ec409-c477-46de-8213-9c3b0236ba3f/381c3ddc8c31479e8422e485805327cd.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/cf8ec409-c477-46de-8213-9c3b0236ba3f/381c3ddc8c31479e8422e485805327cd.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/d08b748f-e58a-4009-8f46-e36697591da8/a4685a179d4e4867a8513744f90dcfd5.txt b/backend/test_uploads/d08b748f-e58a-4009-8f46-e36697591da8/a4685a179d4e4867a8513744f90dcfd5.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/d08b748f-e58a-4009-8f46-e36697591da8/a4685a179d4e4867a8513744f90dcfd5.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/d1b326bd-552b-44f3-b7e6-89b9698ed5e3/1e3d9c252e514446839a9452b1e16b66.txt b/backend/test_uploads/d1b326bd-552b-44f3-b7e6-89b9698ed5e3/1e3d9c252e514446839a9452b1e16b66.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/d1b326bd-552b-44f3-b7e6-89b9698ed5e3/1e3d9c252e514446839a9452b1e16b66.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/d1b326bd-552b-44f3-b7e6-89b9698ed5e3/562113d277a64127b2fc8368f79d2ec4.txt b/backend/test_uploads/d1b326bd-552b-44f3-b7e6-89b9698ed5e3/562113d277a64127b2fc8368f79d2ec4.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/d1b326bd-552b-44f3-b7e6-89b9698ed5e3/562113d277a64127b2fc8368f79d2ec4.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/d1b326bd-552b-44f3-b7e6-89b9698ed5e3/5e55b2e4a0fc42d2b1250485cdedc905.txt b/backend/test_uploads/d1b326bd-552b-44f3-b7e6-89b9698ed5e3/5e55b2e4a0fc42d2b1250485cdedc905.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/d1b326bd-552b-44f3-b7e6-89b9698ed5e3/5e55b2e4a0fc42d2b1250485cdedc905.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/d41cc710-be5b-4635-88af-4116abf76df5/48c2b75dfd0b4118b4788141ccc64f95.txt b/backend/test_uploads/d41cc710-be5b-4635-88af-4116abf76df5/48c2b75dfd0b4118b4788141ccc64f95.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/d41cc710-be5b-4635-88af-4116abf76df5/48c2b75dfd0b4118b4788141ccc64f95.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/e27b141b-e4f2-48fe-9725-f8ea23fb71fa/39248c82845d441089efbc38f33df0e1.txt b/backend/test_uploads/e27b141b-e4f2-48fe-9725-f8ea23fb71fa/39248c82845d441089efbc38f33df0e1.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/e27b141b-e4f2-48fe-9725-f8ea23fb71fa/39248c82845d441089efbc38f33df0e1.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/e6f610d3-e6cd-4266-91a5-e171ccf69bd6/f792f33e837d4092950ffede68e1c6d2.txt b/backend/test_uploads/e6f610d3-e6cd-4266-91a5-e171ccf69bd6/f792f33e837d4092950ffede68e1c6d2.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/e6f610d3-e6cd-4266-91a5-e171ccf69bd6/f792f33e837d4092950ffede68e1c6d2.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/test_uploads/f521c043-30ea-4fe0-9aaf-ff0e8b4eb68d/bb44c3d3727c4bdfab26e54466a0551c.txt b/backend/test_uploads/f521c043-30ea-4fe0-9aaf-ff0e8b4eb68d/bb44c3d3727c4bdfab26e54466a0551c.txt deleted file mode 100644 index 95d09f2b..00000000 --- a/backend/test_uploads/f521c043-30ea-4fe0-9aaf-ff0e8b4eb68d/bb44c3d3727c4bdfab26e54466a0551c.txt +++ /dev/null @@ -1 +0,0 @@ -hello world \ No newline at end of file diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 94f7048a..327c4d15 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -41,12 +41,6 @@ def heartbeat(self): fake_vectorstore.store_chunks = lambda chunks, document_id, filename, user_id: len(chunks) fake_vectorstore.delete_document_chunks = lambda document_id, user_id: None fake_vectorstore.query_chunks = lambda query_embedding, user_id, document_id=None, top_k=10: [] - -if "sentence_transformers" not in sys.modules: - fake_sentence_transformers = types.ModuleType("sentence_transformers") - fake_sentence_transformers.CrossEncoder = lambda *args, **kwargs: None - sys.modules["sentence_transformers"] = fake_sentence_transformers - sys.modules.setdefault("app.rag.vectorstore", fake_vectorstore) slowapi_module = types.ModuleType("slowapi") diff --git a/backend/tests/test_admin.py b/backend/tests/test_admin.py index 68e42b5d..e0297391 100644 --- a/backend/tests/test_admin.py +++ b/backend/tests/test_admin.py @@ -7,7 +7,7 @@ def test_admin_stats_requires_admin(client, auth_headers): response = client.get("/api/v1/admin/stats", headers=auth_headers) assert response.status_code == 403 - assert response.json()["error"]["message"] == "Admin access required" + assert response.json()["detail"] == "Admin access required" def test_admin_stats_returns_aggregate_metrics(client, db_session): diff --git a/backend/tests/test_admin_export.py b/backend/tests/test_admin_export.py deleted file mode 100644 index 2b40187c..00000000 --- a/backend/tests/test_admin_export.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -Unit tests for the secure admin database export endpoint (#437). -""" -import pytest -from fastapi.testclient import TestClient -from app.models import User -from app.auth import create_access_token - - -@pytest.fixture() -def admin_auth_headers(db_session): - """Create a temporary authenticated administrator session context.""" - admin_user = User( - username="root_admin", - email="admin@enterprise.rag", - hashed_password="securepassword", - role="admin", - ) - db_session.add(admin_user) - db_session.commit() - db_session.refresh(admin_user) - token = create_access_token(admin_user.id) - return {"Authorization": f"Bearer {token}"} - - -def test_export_db_enforces_strict_admin_restriction(client: TestClient, auth_headers): - """Ensure standard authenticated non-admin users are strictly rejected with a 403.""" - response = client.get("/api/v1/admin/export-db?format=json", headers=auth_headers) - assert response.status_code == 403 - - -def test_export_db_json_format_success(client: TestClient, admin_auth_headers): - """Verify administrator can pull back entire schema state as an organized JSON object.""" - response = client.get("/api/v1/admin/export-db?format=json", headers=admin_auth_headers) - assert response.status_code == 200 - assert response.headers["content-type"].startswith("application/json") - assert "attachment; filename=db_backup_" in response.headers["content-disposition"] - assert response.headers["x-content-type-options"] == "nosniff" - - data = response.json() - assert isinstance(data, dict) - assert "users" in data - - -def test_export_db_sql_format_success(client: TestClient, admin_auth_headers): - """Verify administrator can pull back sequential structural SQL statements.""" - response = client.get("/api/v1/admin/export-db?format=sql", headers=admin_auth_headers) - assert response.status_code == 200 - assert response.headers["content-type"].startswith("application/sql") - assert "attachment; filename=db_backup_" in response.headers["content-disposition"] - - sql_text = response.text - assert "Database Backup" in sql_text - assert "INSERT INTO" in sql_text - - -def test_export_db_invalid_format_parameter_rejection(client: TestClient, admin_auth_headers): - """Verify endpoint terminates cycle elegantly with a 400 when an unmapped format is requested.""" - response = client.get("/api/v1/admin/export-db?format=yaml", headers=admin_auth_headers) - assert response.status_code == 400 - assert "Invalid export format" in response.json()["detail"] diff --git a/backend/tests/test_agent.py b/backend/tests/test_agent.py index deaf3733..fc5962ee 100644 --- a/backend/tests/test_agent.py +++ b/backend/tests/test_agent.py @@ -144,10 +144,10 @@ def test_generate_answer_stream_error(mock_agent_executor, mock_retriever): assert error_event[0]["data"] == "LLM Down" def test_generate_answer_error(mock_agent_executor, mock_retriever): - from app.exceptions import ExternalServiceException executor, pdf_tool = mock_agent_executor executor.invoke.side_effect = Exception("LLM Down") - with pytest.raises(ExternalServiceException) as exc_info: - generate_answer("test question", "user123", "doc123") - assert "LLM Down" in str(exc_info.value) + result = generate_answer("test question", "user123", "doc123") + + assert "I encountered an error while processing your request:" in result["answer"] + assert "LLM Down" in result["answer"] diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 68d769ea..2a2a4090 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -13,74 +13,3 @@ async def test_health_check_route(): assert response.status_code == 200 # Add more assertions based on expected JSON response - - -async def test_health_check_db_healthy_chroma_healthy(monkeypatch): - # Case 1: DB healthy, Chroma healthy -> status = healthy - # Both are healthy by default in test env (get_db yields test db, get_chroma_client works) - async with AsyncClient(transport=ASGITransport(app=app), base_url="http://testserver") as ac: - response = await ac.get("/health") - assert response.status_code == 200 - data = response.json() - assert data["status"] == "healthy" - assert data["db"] == "up" - assert data["chroma"] == "up" - - -async def test_health_check_db_healthy_chroma_unhealthy(monkeypatch): - # Case 2: DB healthy, Chroma unhealthy -> status = degraded - def mock_get_chroma_client_down(): - raise Exception("Chroma connection refused") - - monkeypatch.setattr("app.main.get_chroma_client", mock_get_chroma_client_down) - - async with AsyncClient(transport=ASGITransport(app=app), base_url="http://testserver") as ac: - response = await ac.get("/health") - assert response.status_code == 200 - data = response.json() - assert data["status"] == "degraded" - assert data["db"] == "up" - assert data["chroma"] == "down" - - -async def test_health_check_db_unhealthy_chroma_healthy(monkeypatch): - # Case 3: DB unhealthy, Chroma healthy -> status = degraded - from sqlalchemy.exc import SQLAlchemyError - - def mock_get_db_down(): - raise SQLAlchemyError("DB Connection refused") - yield - - monkeypatch.setattr("app.main.get_db", mock_get_db_down) - - async with AsyncClient(transport=ASGITransport(app=app), base_url="http://testserver") as ac: - response = await ac.get("/health") - assert response.status_code == 200 - data = response.json() - assert data["status"] == "degraded" - assert data["db"] == "down" - assert data["chroma"] == "up" - - -async def test_health_check_db_unhealthy_chroma_unhealthy(monkeypatch): - # Case 4: DB unhealthy, Chroma unhealthy -> status = unhealthy - from sqlalchemy.exc import SQLAlchemyError - - def mock_get_db_down(): - raise SQLAlchemyError("DB Connection refused") - yield - - def mock_get_chroma_client_down(): - raise Exception("Chroma connection refused") - - monkeypatch.setattr("app.main.get_db", mock_get_db_down) - monkeypatch.setattr("app.main.get_chroma_client", mock_get_chroma_client_down) - - async with AsyncClient(transport=ASGITransport(app=app), base_url="http://testserver") as ac: - response = await ac.get("/health") - assert response.status_code == 200 - data = response.json() - assert data["status"] == "unhealthy" - assert data["db"] == "down" - assert data["chroma"] == "down" - diff --git a/backend/tests/test_auth.py b/backend/tests/test_auth.py index 4cba4e01..372ff0c6 100644 --- a/backend/tests/test_auth.py +++ b/backend/tests/test_auth.py @@ -1,10 +1,3 @@ -from datetime import datetime, timedelta, timezone - -import jwt - -from app.config import get_settings - - VALID_TEST_PASSWORD = "Password1!" @@ -39,14 +32,14 @@ def test_register_duplicate_email_or_username_conflict(client): json={**payload, "username": "anotheruser"}, ) assert duplicate_email.status_code == 409 - assert duplicate_email.json()["error"]["message"] == "Email already registered" + assert duplicate_email.json()["detail"] == "Email already registered" duplicate_username = client.post( "/api/v1/auth/register", json={**payload, "email": "another@example.com"}, ) assert duplicate_username.status_code == 409 - assert duplicate_username.json()["error"]["message"] == "Username already taken" + assert duplicate_username.json()["detail"] == "Username already taken" def test_register_rejects_weak_password(client): @@ -60,8 +53,9 @@ def test_register_rejects_weak_password(client): ) assert response.status_code == 422 - errors = response.json()["error"]["details"]["errors"] - messages = " ".join(item["message"] for item in errors) + detail = response.json()["detail"] + assert isinstance(detail, list) + messages = " ".join(item["msg"] for item in detail) assert "uppercase" in messages.lower() or "8 characters" in messages.lower() @@ -76,8 +70,9 @@ def test_register_rejects_password_missing_special_character(client): ) assert response.status_code == 422 - errors = response.json()["error"]["details"]["errors"] - messages = " ".join(item["message"] for item in errors).lower() + detail = response.json()["detail"] + assert isinstance(detail, list) + messages = " ".join(item["msg"] for item in detail).lower() assert "special character" in messages @@ -101,27 +96,7 @@ def test_login_invalid_password(client, user): ) assert response.status_code == 401 - assert response.json()["error"]["message"] == "Invalid email or password" - - -def test_login_invalid_email(client): - response = client.post( - "/api/v1/auth/login", - json={"email": "missing@example.com", "password": "password123"}, - ) - - assert response.status_code == 401 - assert response.json()["error"]["message"] == "Invalid email or password" - - -def test_auth_me_success(client, auth_headers, user): - response = client.get("/api/v1/auth/me", headers=auth_headers) - - assert response.status_code == 200 - payload = response.json() - assert payload["id"] == str(user.id) - assert payload["username"] == user.username - assert payload["email"] == user.email + assert response.json()["detail"] == "Invalid email or password" def test_auth_me_requires_auth(client): @@ -130,29 +105,6 @@ def test_auth_me_requires_auth(client): assert response.status_code in (401, 403) -def test_auth_me_rejects_expired_token(client, user): - settings = get_settings() - now = datetime.now(timezone.utc) - expired_token = jwt.encode( - { - "sub": str(user.id), - "type": "access", - "exp": now - timedelta(minutes=1), - "iat": now - timedelta(minutes=2), - }, - settings.SECRET_KEY, - algorithm=settings.JWT_ALGORITHM, - ) - - response = client.get( - "/api/v1/auth/me", - headers={"Authorization": f"Bearer {expired_token}"}, - ) - - assert response.status_code == 401 - assert response.json()["detail"] == "Invalid or expired token" - - def test_refresh_token_success(client, refresh_token): response = client.post( "/api/v1/auth/refresh", @@ -216,7 +168,7 @@ def test_update_user_info_rejects_duplicate_email(client, auth_headers, other_us ) assert response.status_code == 400 - assert response.json()["error"]["message"] == "Email already exists" + assert response.json()["detail"] == "Email already exists" from unittest.mock import patch, AsyncMock, MagicMock import urllib.parse @@ -281,7 +233,7 @@ def test_huggingface_callback_invalid_state(client): cookies={"oauth_state": "actual-state"} ) assert response.status_code == 400 - assert "State verification failed" in response.json()["error"]["message"] + assert "State verification failed" in response.json()["detail"] def test_huggingface_logout(client): diff --git a/backend/tests/test_batch_upload.py b/backend/tests/test_batch_upload.py deleted file mode 100644 index 6f1601f8..00000000 --- a/backend/tests/test_batch_upload.py +++ /dev/null @@ -1,227 +0,0 @@ -""" -Tests for POST /api/v1/documents/upload/batch — issue #435. -""" -import io -import uuid -from unittest.mock import MagicMock, patch - -import pytest - -from app.models import Document - - -# ── helpers ────────────────────────────────────────────────────────────────── - -def _fake_txt_file(name: str = "test.txt", content: bytes = b"hello world") -> tuple: - """Return a multipart files tuple accepted by httpx TestClient.""" - return ("files", (name, io.BytesIO(content), "text/plain")) - - -def _patch_validate(monkeypatch, tmp_path, content: bytes = b"hello world") -> None: - """Make validate_upload write content to a real temp file and return its path.""" - import tempfile, shutil - - async def fake_validate(file): - tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".txt", dir=tmp_path) - tmp.write(content) - tmp.close() - return tmp.name - - monkeypatch.setattr("app.routes.documents.validate_upload", fake_validate) - - -def _patch_celery(monkeypatch) -> MagicMock: - """Stub out Celery so tests never touch Redis.""" - mock_task = MagicMock() - mock_task.id = f"celery-{uuid.uuid4().hex}" - monkeypatch.setattr( - "app.routes.documents.process_document", - MagicMock(delay=MagicMock(return_value=mock_task)), - ) - return mock_task - - -# ── tests ───────────────────────────────────────────────────────────────────── - -def test_batch_upload_single_file(client, auth_headers, monkeypatch, tmp_path): - _patch_validate(monkeypatch, tmp_path) - _patch_celery(monkeypatch) - - response = client.post( - "/api/v1/documents/upload/batch", - headers=auth_headers, - files=[_fake_txt_file("report.txt")], - data={"chunk_size": "1000", "chunk_overlap": "200"}, - ) - - assert response.status_code == 202 - payload = response.json() - assert payload["total"] == 1 - assert len(payload["documents"]) == 1 - assert payload["documents"][0]["original_name"] == "report.txt" - assert payload["documents"][0]["status"] == "pending" - assert len(payload["task_ids"]) == 1 - assert payload["failed"] == [] - - -def test_batch_upload_multiple_files(client, auth_headers, monkeypatch, tmp_path): - _patch_validate(monkeypatch, tmp_path) - _patch_celery(monkeypatch) - - response = client.post( - "/api/v1/documents/upload/batch", - headers=auth_headers, - files=[ - _fake_txt_file("a.txt"), - _fake_txt_file("b.txt"), - _fake_txt_file("c.txt"), - ], - data={"chunk_size": "1000", "chunk_overlap": "200"}, - ) - - assert response.status_code == 202 - payload = response.json() - assert payload["total"] == 3 - assert len(payload["documents"]) == 3 - assert len(payload["task_ids"]) == 3 - assert payload["failed"] == [] - - -def test_batch_upload_rejects_bad_extension(client, auth_headers, monkeypatch, tmp_path): - """A .exe file should land in failed[], not crash the whole batch.""" - _patch_validate(monkeypatch, tmp_path) - _patch_celery(monkeypatch) - - response = client.post( - "/api/v1/documents/upload/batch", - headers=auth_headers, - files=[ - _fake_txt_file("good.txt"), - ("files", ("bad.exe", io.BytesIO(b"binary"), "application/octet-stream")), - ], - data={"chunk_size": "1000", "chunk_overlap": "200"}, - ) - - assert response.status_code == 202 - payload = response.json() - assert payload["total"] == 1 - assert payload["documents"][0]["original_name"] == "good.txt" - assert "bad.exe" in payload["failed"] - - -def test_batch_upload_all_files_fail_returns_400(client, auth_headers, monkeypatch, tmp_path): - """When every file fails, the endpoint should return 400.""" - _patch_validate(monkeypatch, tmp_path) - _patch_celery(monkeypatch) - - response = client.post( - "/api/v1/documents/upload/batch", - headers=auth_headers, - files=[ - ("files", ("bad1.exe", io.BytesIO(b"x"), "application/octet-stream")), - ("files", ("bad2.exe", io.BytesIO(b"y"), "application/octet-stream")), - ], - data={"chunk_size": "1000", "chunk_overlap": "200"}, - ) - - assert response.status_code == 400 - - -def test_batch_upload_requires_auth(client): - response = client.post( - "/api/v1/documents/upload/batch", - files=[_fake_txt_file("test.txt")], - data={"chunk_size": "1000", "chunk_overlap": "200"}, - ) - - assert response.status_code in (401, 403) - - -def test_batch_upload_invalid_chunk_size(client, auth_headers, monkeypatch, tmp_path): - _patch_validate(monkeypatch, tmp_path) - _patch_celery(monkeypatch) - - response = client.post( - "/api/v1/documents/upload/batch", - headers=auth_headers, - files=[_fake_txt_file("test.txt")], - data={"chunk_size": "50", "chunk_overlap": "10"}, - ) - - assert response.status_code == 400 - - -def test_batch_upload_invalid_chunk_overlap(client, auth_headers, monkeypatch, tmp_path): - _patch_validate(monkeypatch, tmp_path) - _patch_celery(monkeypatch) - - response = client.post( - "/api/v1/documents/upload/batch", - headers=auth_headers, - files=[_fake_txt_file("test.txt")], - data={"chunk_size": "500", "chunk_overlap": "600"}, - ) - - assert response.status_code == 400 - - -def test_batch_upload_celery_fallback_uses_background_task(client, auth_headers, monkeypatch, tmp_path): - """When Celery is unavailable, tasks should fall back gracefully.""" - _patch_validate(monkeypatch, tmp_path) - - # Make Celery raise so the fallback branch is taken - monkeypatch.setattr( - "app.routes.documents.process_document", - MagicMock(delay=MagicMock(side_effect=Exception("Redis down"))), - ) - - response = client.post( - "/api/v1/documents/upload/batch", - headers=auth_headers, - files=[_fake_txt_file("fallback.txt")], - data={"chunk_size": "1000", "chunk_overlap": "200"}, - ) - - assert response.status_code == 202 - payload = response.json() - assert payload["total"] == 1 - assert payload["task_ids"][0].startswith("local_") - - -def test_batch_upload_document_persisted_in_db(client, auth_headers, monkeypatch, tmp_path, db_session): - _patch_validate(monkeypatch, tmp_path) - _patch_celery(monkeypatch) - - response = client.post( - "/api/v1/documents/upload/batch", - headers=auth_headers, - files=[_fake_txt_file("persisted.txt")], - data={"chunk_size": "1000", "chunk_overlap": "200"}, - ) - - assert response.status_code == 202 - doc_id = response.json()["documents"][0]["id"] - doc = db_session.get(Document, doc_id) - assert doc is not None - assert doc.original_name == "persisted.txt" - assert doc.status == "pending" - assert doc.chunk_size == 1000 - assert doc.chunk_overlap == 200 - - -def test_batch_upload_chunk_settings_stored(client, auth_headers, monkeypatch, tmp_path, db_session): - _patch_validate(monkeypatch, tmp_path) - _patch_celery(monkeypatch) - - response = client.post( - "/api/v1/documents/upload/batch", - headers=auth_headers, - files=[_fake_txt_file("chunked.txt")], - data={"chunk_size": "800", "chunk_overlap": "100"}, - ) - - assert response.status_code == 202 - doc_id = response.json()["documents"][0]["id"] - doc = db_session.get(Document, doc_id) - assert doc.chunk_size == 800 - assert doc.chunk_overlap == 100 diff --git a/backend/tests/test_bm25.py b/backend/tests/test_bm25.py deleted file mode 100644 index 01356361..00000000 --- a/backend/tests/test_bm25.py +++ /dev/null @@ -1,50 +0,0 @@ -import pytest -import os -from app.rag import bm25 - -def test_tokenize(): - # Test that punctuation is stripped and tokens are lowercased - text = "This is a document." - assert bm25.tokenize(text) == ["this", "is", "a", "document"] - - text_qwen = "The model used is Qwen." - assert bm25.tokenize(text_qwen) == ["the", "model", "used", "is", "qwen"] - - text_mixed = "BM25 keyword-search module, testing!" - assert bm25.tokenize(text_mixed) == ["bm25", "keyword", "search", "module", "testing"] - -def test_store_and_query_bm25(tmp_path, monkeypatch): - # Mock settings.CHROMA_PERSIST_DIR to use the temp path - monkeypatch.setattr(bm25.settings, "CHROMA_PERSIST_DIR", str(tmp_path)) - - user_id = "test-user-123" - doc_id = "test-doc-abc" - chunks = [ - {"text": "The model used is Qwen.", "page": 1}, - {"text": "BM25 retrieval performs keyword search.", "page": 2}, - {"text": "This is a completely unrelated third document chunk.", "page": 3}, - ] - - # Store index - bm25.store_bm25_index(chunks, doc_id, "test.pdf", user_id) - - # Check that the index file was created - expected_path = bm25.get_bm25_path(user_id, doc_id) - assert os.path.exists(expected_path) - - # Query with exact word (originally failed because of punctuation) - # The first chunk has "Qwen." (with period), query is "qwen" - results = bm25.query_bm25("qwen", user_id, doc_id, top_k=1) - assert len(results) == 1 - assert "Qwen" in results[0]["text"] - assert results[0]["page"] == 1 - - # Query all indexes for user - results_all = bm25.query_bm25("keyword", user_id, top_k=2) - assert len(results_all) == 1 - assert "BM25 retrieval" in results_all[0]["text"] - assert results_all[0]["page"] == 2 - - # Delete index - bm25.delete_bm25_index(doc_id, user_id) - assert not os.path.exists(expected_path) diff --git a/backend/tests/test_cache.py b/backend/tests/test_cache.py deleted file mode 100644 index 0db4b73e..00000000 --- a/backend/tests/test_cache.py +++ /dev/null @@ -1,110 +0,0 @@ -""" -Unit tests for the response caching utility (Issue #45, #640). -Run with: pytest backend/tests/test_cache.py -v -""" - -import pytest -import app.cache as cache_module - - -@pytest.fixture(autouse=True) -def reset_cache(): - """Clear all cache state before each test so tests are independent.""" - cache_module._lru_store.clear() - cache_module._lru_order.clear() - cache_module._redis_available = False - cache_module._redis_client = None - yield - - -def test_cache_miss_returns_none(): - result = cache_module.get_cached_response("user1", "doc123", "What is this about?") - assert result is None - - -def test_set_and_get_roundtrip(): - cache_module.set_cached_response("user1", "doc123", "What is this?", "It is a test.") - result = cache_module.get_cached_response("user1", "doc123", "What is this?") - assert result == "It is a test." - - -def test_different_documents_do_not_collide(): - cache_module.set_cached_response("user1", "doc_A", "Same question?", "Answer A") - cache_module.set_cached_response("user1", "doc_B", "Same question?", "Answer B") - assert cache_module.get_cached_response("user1", "doc_A", "Same question?") == "Answer A" - assert cache_module.get_cached_response("user1", "doc_B", "Same question?") == "Answer B" - - -def test_question_normalised_to_lowercase(): - """Cache should match regardless of question casing.""" - cache_module.set_cached_response("user1", "doc1", "What is AI?", "AI is artificial intelligence.") - result = cache_module.get_cached_response("user1", "doc1", "WHAT IS AI?") - assert result == "AI is artificial intelligence." - - -def test_invalidate_removes_entry(): - cache_module.set_cached_response("user1", "doc1", "Hello?", "Hi!") - cache_module.invalidate_cache("user1", "doc1", "Hello?") - assert cache_module.get_cached_response("user1", "doc1", "Hello?") is None - - -def test_lru_eviction_removes_oldest(): - """When LRU reaches max size, the oldest entry is evicted.""" - cache_module.LRU_MAX_SIZE = 3 - cache_module.set_cached_response("user1", "doc", "Q1", "A1") - cache_module.set_cached_response("user1", "doc", "Q2", "A2") - cache_module.set_cached_response("user1", "doc", "Q3", "A3") - cache_module.set_cached_response("user1", "doc", "Q4", "A4") # should evict Q1 - assert cache_module.get_cached_response("user1", "doc", "Q1") is None - assert cache_module.get_cached_response("user1", "doc", "Q4") == "A4" - - -def test_make_cache_key_is_deterministic(): - k1 = cache_module.make_cache_key("user1", "doc1", "What is this?") - k2 = cache_module.make_cache_key("user1", "doc1", "What is this?") - assert k1 == k2 - - -def test_make_cache_key_differs_by_document(): - k1 = cache_module.make_cache_key("user1", "doc1", "Same question") - k2 = cache_module.make_cache_key("user1", "doc2", "Same question") - assert k1 != k2 - - -def test_make_cache_key_is_64_chars(): - """SHA-256 hex digest is always exactly 64 characters.""" - key = cache_module.make_cache_key("user1", "any_doc", "any question") - assert len(key) == 64 - - -# ── Cross-user isolation ──────────────────────────────────────────── -# make_cache_key omitted user_id entirely, so two different users asking the -# identical question with no document_id (cross-document RAG over their own -# private knowledge base) collapsed onto the same cache entry — meaning the -# second user's request returned the first user's privately-generated answer. - -def test_make_cache_key_differs_by_user(): - """Same document + question, different users, must produce different keys.""" - k1 = cache_module.make_cache_key("user-a", "doc1", "Summarize the key points") - k2 = cache_module.make_cache_key("user-b", "doc1", "Summarize the key points") - assert k1 != k2 - - -def test_make_cache_key_differs_by_user_with_no_document(): - """The empty-document_id case from ask_question's `str(payload.document_id or "")` - is exactly where the original bug collapsed two different users onto one key.""" - k1 = cache_module.make_cache_key("user-a", "", "What does this say about pricing?") - k2 = cache_module.make_cache_key("user-b", "", "What does this say about pricing?") - assert k1 != k2 - - -def test_documentless_query_cache_is_isolated_per_user(): - """Regression test for #640: one user's cached document-less RAG answer - must never be served back to a different user asking the same question.""" - cache_module.set_cached_response("user-a", "", "summarize the key points", "User A's private summary") - - # A different user asking the identical normalized question text must miss. - assert cache_module.get_cached_response("user-b", "", "summarize the key points") is None - - # The original user still gets their own cached answer back. - assert cache_module.get_cached_response("user-a", "", "summarize the key points") == "User A's private summary" \ No newline at end of file diff --git a/backend/tests/test_celery_ingestion.py b/backend/tests/test_celery_ingestion.py deleted file mode 100644 index 955059e9..00000000 --- a/backend/tests/test_celery_ingestion.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Regression tests for the process_document Celery task. - -These guard against the task body becoming a no-op stub again (see issue -#635): process_document must actually drive the real ingestion pipeline in -app.services.document_ingestion, not just flip the document's status to -"ready" without storing any chunks/vectors. -""" -import pytest -from unittest.mock import patch, MagicMock - -from app.models import Document -from app.tasks import process_document - - -def _make_pending_document(db_session, doc_id="test-doc-123", user_id="user-456"): - test_doc = Document( - id=doc_id, - filename="sample.pdf", - original_name="sample.pdf", - status="pending", - user_id=user_id, - ) - db_session.add(test_doc) - db_session.commit() - return test_doc - - -@pytest.fixture() -def patched_session_factory(db_session, monkeypatch): - """Route both app.tasks (get_db_session) and document_ingestion - (SessionLocal) onto the same test db_session, so the task and the - pipeline it dispatches into observe a single consistent view of the row. - """ - from contextlib import contextmanager - - @contextmanager - def _fake_get_db_session(): - yield db_session - - monkeypatch.setattr("app.tasks.get_db_session", _fake_get_db_session) - monkeypatch.setattr("app.database.SessionLocal", lambda: db_session) - monkeypatch.setattr(db_session, "close", lambda: None) - - return db_session - - -def test_process_document_runs_real_ingestion_pipeline(patched_session_factory): - """process_document must delegate into ingest_document and persist real - chunk/vector results - not silently no-op and mark the doc 'ready' with - zero chunks (the bug in #635). - """ - db_session = patched_session_factory - _make_pending_document(db_session) - - fake_chunks = [ - {"text": "first chunk", "page": 1, "type": "text"}, - {"text": "second chunk", "page": 1, "type": "text"}, - {"text": "third chunk", "page": 2, "type": "text"}, - ] - - with patch("app.services.document_ingestion.get_page_count", return_value=2), \ - patch("app.services.document_ingestion.chunk_document", return_value=fake_chunks), \ - patch("app.services.document_ingestion.store_chunks", return_value=len(fake_chunks)) as mock_store, \ - patch("app.services.document_ingestion.persist_document_keywords"): - - task_result = process_document.apply( - kwargs={ - "document_id": "test-doc-123", - "filepath": "/tmp/sample.pdf", - "original_name": "sample.pdf", - "user_id": "user-456", - } - ) - - assert task_result.status == "SUCCESS" - assert task_result.result == {"document_id": "test-doc-123", "status": "ready"} - - mock_store.assert_called_once() - call_kwargs = mock_store.call_args.kwargs - assert call_kwargs["chunks"] == fake_chunks - assert call_kwargs["document_id"] == "test-doc-123" - - updated_doc = db_session.query(Document).filter_by(id="test-doc-123").first() - assert updated_doc is not None - assert updated_doc.status == "ready" - assert updated_doc.chunk_count == 3 - assert updated_doc.page_count == 2 - assert updated_doc.retry_count == 1 - - -def test_process_document_marks_failed_when_no_text_extracted(patched_session_factory): - """If ingestion legitimately fails (e.g. no extractable text), - process_document must surface that as a failed task/status rather than - reporting 'ready' regardless of pipeline outcome. - """ - db_session = patched_session_factory - _make_pending_document(db_session, doc_id="test-doc-empty") - - with patch("app.services.document_ingestion.get_page_count", return_value=1), \ - patch("app.services.document_ingestion.chunk_document", return_value=[]): - - task_result = process_document.apply( - kwargs={ - "document_id": "test-doc-empty", - "filepath": "/tmp/empty.pdf", - "original_name": "empty.pdf", - "user_id": "user-456", - } - ) - - assert task_result.status == "FAILURE" - - updated_doc = db_session.query(Document).filter_by(id="test-doc-empty").first() - assert updated_doc is not None - assert updated_doc.status == "failed" - assert updated_doc.chunk_count == 0 \ No newline at end of file diff --git a/backend/tests/test_chat.py b/backend/tests/test_chat.py index 1a6b2c14..26e143a5 100644 --- a/backend/tests/test_chat.py +++ b/backend/tests/test_chat.py @@ -36,7 +36,7 @@ def test_chat_ask_document_not_found(client, auth_headers): ) assert response.status_code == 404 - assert response.json()["error"]["message"] == "Document not found" + assert response.json()["detail"] == "Document not found" def test_chat_ask_document_not_ready(client, auth_headers, pending_document): @@ -47,7 +47,7 @@ def test_chat_ask_document_not_ready(client, auth_headers, pending_document): ) assert response.status_code == 400 - assert "Document is still pending" in response.json()["error"]["message"] + assert "Document is still pending" in response.json()["detail"] def test_chat_ask_blocks_prompt_injection_before_generation(client, auth_headers, ready_document, monkeypatch): @@ -70,7 +70,7 @@ def fake_generate_answer(*_args, **_kwargs): ) assert response.status_code == 400 - assert "prompt-injection" in response.json()["error"]["message"] + assert "prompt-injection" in response.json()["detail"] assert called is False @@ -94,7 +94,7 @@ def fake_generate_answer_stream(*_args, **_kwargs): ) assert response.status_code == 400 - assert "prompt-injection" in response.json()["error"]["message"] + assert "prompt-injection" in response.json()["detail"] assert called is False @@ -127,118 +127,3 @@ class MockResponse: generate_answer(question="hello?", user_id="some-user", hf_token=None) from app.config import get_settings assert called_with_token == get_settings().HF_TOKEN - - -def test_clear_chat_history_with_shared_messages(client, auth_headers, ready_document, db_session, user): - from app.models import ChatMessage, SharedMessage - - # Create a user ChatMessage and an assistant ChatMessage associated with ready_document - user_msg = ChatMessage( - user_id=user.id, - document_id=ready_document.id, - role="user", - content="Hello, is anyone there?", - ) - assistant_msg = ChatMessage( - user_id=user.id, - document_id=ready_document.id, - role="assistant", - content="Yes, I am here.", - ) - db_session.add_all([user_msg, assistant_msg]) - db_session.commit() - db_session.refresh(assistant_msg) - - # Make assistant message shared by creating a SharedMessage link - shared = SharedMessage(message_id=assistant_msg.id) - db_session.add(shared) - db_session.commit() - - assistant_msg_id = assistant_msg.id - - # Expunge objects so session doesn't try to auto-refresh deleted rows - db_session.expunge(user_msg) - db_session.expunge(assistant_msg) - db_session.expunge(shared) - - # Call DELETE /api/v1/chat/history/{document_id} - response = client.delete( - f"/api/v1/chat/history/{ready_document.id}", - headers=auth_headers, - ) - - # Check results - assert response.status_code == 200 - assert response.json() == {"message": "Chat history cleared"} - - # Check that ChatMessage records are deleted - remaining_messages = db_session.query(ChatMessage).filter( - ChatMessage.document_id == ready_document.id - ).all() - assert len(remaining_messages) == 0 - - # Check that SharedMessage records are deleted - remaining_shared = db_session.query(SharedMessage).filter( - SharedMessage.message_id == assistant_msg_id - ).all() - assert len(remaining_shared) == 0 - - -def test_clear_chat_history_repeated_or_empty(client, auth_headers, ready_document, db_session): - from app.models import ChatMessage - # Check history is empty initially - remaining_messages = db_session.query(ChatMessage).filter( - ChatMessage.document_id == ready_document.id - ).all() - assert len(remaining_messages) == 0 - - # First delete on empty history - response = client.delete( - f"/api/v1/chat/history/{ready_document.id}", - headers=auth_headers, - ) - assert response.status_code == 200 - assert response.json() == {"message": "Chat history cleared"} - - # Second delete (repeated request) - response = client.delete( - f"/api/v1/chat/history/{ready_document.id}", - headers=auth_headers, - ) - assert response.status_code == 200 - assert response.json() == {"message": "Chat history cleared"} - - -def test_chat_ws_rate_limited_after_threshold(client, user, monkeypatch): - """ - Regression test for #639: /chat/ws must enforce the same - CHAT_QUERY_RATE_LIMIT (15/minute) that @limiter.limit applies to - POST /chat/ask and /chat/ask/stream, instead of letting an unbounded - number of RAG/LLM pipeline calls through per user over the WebSocket - transport — including across multiple separate connections. - """ - from app.auth import create_access_token - from app.rate_limit import CHAT_QUERY_RATE_LIMIT - - def fake_generate_answer_stream(*_args, **_kwargs): - yield "data: {}\n\n" - - monkeypatch.setattr("app.routes.chat.generate_answer_stream", fake_generate_answer_stream) - - token = create_access_token(user.id) - limit = int(CHAT_QUERY_RATE_LIMIT.split("/")[0]) - - for _ in range(limit): - with client.websocket_connect(f"/api/v1/chat/ws?token={token}") as ws: - ws.send_json({"question": "What is in the doc?"}) - seen_types = [] - while "done" not in seen_types: - msg = ws.receive_json() - seen_types.append(msg.get("type")) - assert "error" not in seen_types - - # The connection beyond the configured limit must be rejected before - # generate_answer_stream runs again, without even waiting for a payload. - with client.websocket_connect(f"/api/v1/chat/ws?token={token}") as ws: - msg = ws.receive_json() - assert msg == {"type": "error", "data": "Rate limit exceeded"} diff --git a/backend/tests/test_chunk_settings_concurrency.py b/backend/tests/test_chunk_settings_concurrency.py deleted file mode 100644 index c1892a82..00000000 --- a/backend/tests/test_chunk_settings_concurrency.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Regression tests for re-chunking a document while it is -still processing must not queue a second concurrent ingestion run. - -update_chunk_settings previously reset doc.status to "pending" and -re-queued process_document.delay(...) unconditionally, with no check on -the document's current status. If a prior ingestion run for the same -document_id was still "processing", this let two ingestion runs execute -concurrently against the same document - and since store_chunks() in -vectorstore.py performs a non-atomic delete-then-batch-insert sequence, -the two runs could interleave and corrupt the vector store relative to -whatever chunk_count ends up persisted in Postgres. -""" -from app.models import Document - - -def test_update_chunk_settings_rejects_while_document_processing( - client, auth_headers, db_session, user, monkeypatch -): - """A document mid-ingestion must reject a re-chunk request with 409, - not silently reset its status and re-queue a second concurrent run. - """ - document = Document( - user_id=user.id, - filename="processing.pdf", - original_name="processing.pdf", - file_size=256, - status="processing", - chunk_count=0, - page_count=0, - ) - db_session.add(document) - db_session.commit() - db_session.refresh(document) - - def _delay_should_not_be_called(*args, **kwargs): - raise AssertionError( - "process_document.delay was called for a document still " - "processing - the concurrency guard did not reject the request" - ) - - monkeypatch.setattr( - "app.routes.documents.process_document.delay", - _delay_should_not_be_called, - ) - - response = client.post( - f"/api/v1/documents/{document.id}/chunk_settings", - json={"chunk_size": 500, "chunk_overlap": 50}, - headers=auth_headers, - ) - - assert response.status_code == 409 - - refreshed = db_session.get(Document, document.id) - assert refreshed.status == "processing" - assert refreshed.chunk_count == 0 - assert refreshed.page_count == 0 - - -def test_update_chunk_settings_allows_when_not_processing( - client, auth_headers, ready_document, db_session, monkeypatch -): - """A document that is "ready" (i.e. not mid-ingestion) must still be - allowed to re-chunk - the guard should only block "processing". - """ - queued = {} - - class _FakeTask: - id = "fake-task-id" - - def _fake_delay(**kwargs): - queued.update(kwargs) - return _FakeTask() - - monkeypatch.setattr("app.routes.documents.process_document.delay", _fake_delay) - - response = client.post( - f"/api/v1/documents/{ready_document.id}/chunk_settings", - json={"chunk_size": 500, "chunk_overlap": 50}, - headers=auth_headers, - ) - - assert response.status_code == 200 - assert queued["document_id"] == ready_document.id - - refreshed = db_session.get(Document, ready_document.id) - assert refreshed.status == "pending" - assert refreshed.chunk_size == 500 - assert refreshed.chunk_overlap == 50 \ No newline at end of file diff --git a/backend/tests/test_chunker.py b/backend/tests/test_chunker.py index 4a635530..85575539 100644 --- a/backend/tests/test_chunker.py +++ b/backend/tests/test_chunker.py @@ -49,14 +49,12 @@ def test_table_to_markdown_cleans_cells_and_escapes_pipes(): ["Ravi", 28], ] - assert _table_to_markdown(rows) == "\n".join( - [ - "| Name | Age | Role |", - "| --- | --- | --- |", - "| Asha Rao | 24 | Admin \\| Owner |", - "| Ravi | 28 | |", - ] - ) + assert _table_to_markdown(rows) == "\n".join([ + "| Name | Age | Role |", + "| --- | --- | --- |", + "| Asha Rao | 24 | Admin \\| Owner |", + "| Ravi | 28 | |", + ]) def test_pdf_table_detection_separates_table_from_paragraph(monkeypatch): @@ -75,12 +73,12 @@ def find_tables(self): def extract_words(self): return [ - {"text": "Intro", "x0": 40, "x1": 70, "top": 20, "bottom": 30}, - {"text": "paragraph", "x0": 75, "x1": 140, "top": 20, "bottom": 30}, - {"text": "Name", "x0": 45, "x1": 80, "top": 100, "bottom": 110}, + {"text": "Intro", "x0": 40, "x1": 70, "top": 20, "bottom": 30}, + {"text": "paragraph", "x0": 75, "x1": 140, "top": 20, "bottom": 30}, + {"text": "Name", "x0": 45, "x1": 80, "top": 100, "bottom": 110}, {"text": "Amount", "x0": 160, "x1": 220, "top": 100, "bottom": 110}, - {"text": "Alpha", "x0": 45, "x1": 85, "top": 125, "bottom": 135}, - {"text": "$10", "x0": 160, "x1": 185, "top": 125, "bottom": 135}, + {"text": "Alpha", "x0": 45, "x1": 85, "top": 125, "bottom": 135}, + {"text": "$10", "x0": 160, "x1": 185, "top": 125, "bottom": 135}, ] class FakePdf: @@ -106,419 +104,3 @@ def __exit__(self, exc_type, exc, traceback): assert chunks[1]["bbox"] == "[0.1, 0.45, 0.75, 0.8]" assert "| Name | Amount |" in chunks[1]["text"] assert "| Alpha | $10 |" in chunks[1]["text"] - - -def test_unstructured_table_detection(monkeypatch): - # Create fake Unstructured Table and Text element classes - class FakeTableClass: - pass - - class FakeTable(FakeTableClass): - def __init__(self): - self.rows = [["Name", "Amount"], ["Delta", "$40"]] - self.page_number = 3 - - class FakeText: - def __init__(self): - self.text = "Intro paragraph" - self.page_number = 3 - - def fake_partition_pdf(filename): - return [FakeText(), FakeTable()] - - # Insert fake unstructured modules - monkeypatch.setitem(sys.modules, "unstructured", types.SimpleNamespace()) - monkeypatch.setitem(sys.modules, "unstructured.partition", types.SimpleNamespace()) - monkeypatch.setitem( - sys.modules, - "unstructured.partition.pdf", - types.SimpleNamespace(partition_pdf=fake_partition_pdf), - ) - monkeypatch.setitem(sys.modules, "unstructured.documents", types.SimpleNamespace()) - monkeypatch.setitem( - sys.modules, - "unstructured.documents.elements", - types.SimpleNamespace(Table=FakeTableClass), - ) - - monkeypatch.setattr(chunker, "extract_pdf_images", lambda _filepath: []) - - chunks = chunk_document("sample.pdf") - - # Expect two chunks: text then table - assert len(chunks) >= 2 - assert chunks[0]["chunk_type"] == "text" - assert "Intro paragraph" in chunks[0]["text"] - # find a table chunk - table_chunks = [c for c in chunks if c.get("chunk_type") == "table"] - assert table_chunks, "No table chunks produced by Unstructured path" - assert table_chunks[0]["page"] == 3 - assert "| Name | Amount |" in table_chunks[0]["text"] - assert "| Delta | $40 |" in table_chunks[0]["text"] - - -# ── _table_to_markdown edge cases ──────────────────────────────────────────── - - -def test_table_to_markdown_empty_rows_returns_empty(): - assert _table_to_markdown([]) == "" - - -def test_table_to_markdown_all_blank_cells_returns_empty(): - rows = [[None, None], [" ", ""], [None, " "]] - assert _table_to_markdown(rows) == "" - - -def test_table_to_markdown_single_row_acts_as_header(): - rows = [["Product", "Price"]] - result = _table_to_markdown(rows) - assert result == "\n".join( - [ - "| Product | Price |", - "| --- | --- |", - ] - ) - - -def test_table_to_markdown_ragged_rows_padded_to_max_width(): - """Rows shorter than the widest row must be right-padded with empty strings.""" - rows = [ - ["A", "B", "C"], - ["X"], - ["Y", "Z"], - ] - result = _table_to_markdown(rows) - lines = result.splitlines() - # Every line should have the same number of pipe characters - pipe_counts = [line.count("|") for line in lines] - assert len(set(pipe_counts)) == 1, "All rows must have equal column count" - - -def test_table_to_markdown_whitespace_normalised_in_cells(): - rows = [["Col\t1", "Col\n2"], ["val a", "val\tb"]] - result = _table_to_markdown(rows) - assert "Col 1" in result - assert "Col 2" in result - assert "val a" in result - assert "val b" in result - - -def test_table_to_markdown_pipe_in_cell_is_escaped(): - rows = [["A|B", "C"], ["x|y|z", "w"]] - result = _table_to_markdown(rows) - assert "A\\|B" in result - assert "x\\|y\\|z" in result - - -def test_table_to_markdown_separator_row_uses_triple_dash(): - rows = [["H1", "H2"], ["v1", "v2"]] - lines = _table_to_markdown(rows).splitlines() - assert lines[1] == "| --- | --- |" - - -# ── pdfplumber path — multi-page ───────────────────────────────────────────── - - -def test_pdf_table_multi_page_produces_chunk_per_page(monkeypatch): - """Tables on different pages must produce separate table chunks with correct page numbers.""" - - class FakeTable: - def __init__(self, page_num): - self._page_num = page_num - self.bbox = (0, 50, 200, 150) - - def extract(self): - return [["Item", "Qty"], [f"Row-p{self._page_num}", "1"]] - - class FakePage: - def __init__(self, page_num): - self._page_num = page_num - self.width = 200 - self.height = 200 - - def find_tables(self): - return [FakeTable(self._page_num)] - - def extract_words(self): - # No paragraph words — all words are inside the table bbox - return [] - - class FakePdf: - pages = [FakePage(1), FakePage(2)] - - def __enter__(self): - return self - - def __exit__(self, *_): - return False - - fake_pdfplumber = types.SimpleNamespace(open=lambda _: FakePdf()) - monkeypatch.setitem(sys.modules, "pdfplumber", fake_pdfplumber) - monkeypatch.setattr(chunker, "extract_pdf_images", lambda _: []) - - chunks = chunk_document("multipage.pdf") - - table_chunks = [c for c in chunks if c.get("chunk_type") == "table"] - assert len(table_chunks) == 2 - assert table_chunks[0]["page"] == 1 - assert table_chunks[1]["page"] == 2 - assert "Row-p1" in table_chunks[0]["text"] - assert "Row-p2" in table_chunks[1]["text"] - - -def test_pdf_empty_table_is_not_emitted(monkeypatch): - """A table whose cells are all blank must produce no chunk.""" - - class FakeEmptyTable: - bbox = (0, 50, 200, 150) - - def extract(self): - return [[None, ""], [" ", None]] - - class FakePage: - width = 200 - height = 200 - - def find_tables(self): - return [FakeEmptyTable()] - - def extract_words(self): - return [] - - class FakePdf: - pages = [FakePage()] - - def __enter__(self): - return self - - def __exit__(self, *_): - return False - - fake_pdfplumber = types.SimpleNamespace(open=lambda _: FakePdf()) - monkeypatch.setitem(sys.modules, "pdfplumber", fake_pdfplumber) - monkeypatch.setattr(chunker, "extract_pdf_images", lambda _: []) - - chunks = chunk_document("empty_table.pdf") - table_chunks = [c for c in chunks if c.get("chunk_type") == "table"] - assert table_chunks == [], "Empty tables must not produce chunks" - - -def test_pdf_table_index_increments_per_page(monkeypatch): - """table_index must restart at 0 for each page (pdfplumber path).""" - - class FakeTable: - bbox = (0, 50, 200, 150) - - def extract(self): - return [["H"], ["V"]] - - class FakePage: - width = 200 - height = 200 - - def find_tables(self): - return [FakeTable(), FakeTable()] - - def extract_words(self): - return [] - - class FakePdf: - pages = [FakePage()] - - def __enter__(self): - return self - - def __exit__(self, *_): - return False - - fake_pdfplumber = types.SimpleNamespace(open=lambda _: FakePdf()) - monkeypatch.setitem(sys.modules, "pdfplumber", fake_pdfplumber) - monkeypatch.setattr(chunker, "extract_pdf_images", lambda _: []) - - chunks = chunk_document("two_tables.pdf") - table_chunks = [c for c in chunks if c.get("chunk_type") == "table"] - assert len(table_chunks) == 2 - assert table_chunks[0]["table_index"] == 0 - assert table_chunks[1]["table_index"] == 1 - - -def test_pdf_table_bbox_normalised_to_unit_range(monkeypatch): - """Stored bbox values must each be within [0.0, 1.0].""" - - class FakeTable: - bbox = (20, 40, 180, 160) - - def extract(self): - return [["X", "Y"], ["1", "2"]] - - class FakePage: - width = 200 - height = 200 - - def find_tables(self): - return [FakeTable()] - - def extract_words(self): - return [] - - class FakePdf: - pages = [FakePage()] - - def __enter__(self): - return self - - def __exit__(self, *_): - return False - - import json as _json - - fake_pdfplumber = types.SimpleNamespace(open=lambda _: FakePdf()) - monkeypatch.setitem(sys.modules, "pdfplumber", fake_pdfplumber) - monkeypatch.setattr(chunker, "extract_pdf_images", lambda _: []) - - chunks = chunk_document("bbox_check.pdf") - table_chunks = [c for c in chunks if c.get("chunk_type") == "table"] - assert table_chunks, "Expected at least one table chunk" - - bbox = _json.loads(table_chunks[0]["bbox"]) - assert len(bbox) == 4 - for val in bbox: - assert 0.0 <= val <= 1.0, f"bbox value {val} out of [0, 1] range" - -# ── chunk_index continuity ──────────────────────────────────────────────────── - - -def test_chunk_index_is_monotonically_increasing(monkeypatch): - """chunk_index must be a 0-based counter that never resets or skips mid-document.""" - - class FakeTable: - bbox = (0, 50, 200, 150) - - def extract(self): - return [["H1", "H2"], ["r1", "r2"]] - - class FakePage: - width = 200 - height = 200 - - def find_tables(self): - return [FakeTable()] - - def extract_words(self): - return [ - {"text": "Intro", "x0": 0, "x1": 40, "top": 10, "bottom": 20}, - ] - - class FakePdf: - pages = [FakePage()] - - def __enter__(self): - return self - - def __exit__(self, *_): - return False - - fake_pdfplumber = types.SimpleNamespace(open=lambda _: FakePdf()) - monkeypatch.setitem(sys.modules, "pdfplumber", fake_pdfplumber) - monkeypatch.setattr(chunker, "extract_pdf_images", lambda _: []) - - chunks = chunk_document("index_check.pdf") - indices = [c["chunk_index"] for c in chunks] - - assert indices == list( - range(len(indices)) - ), f"chunk_index must be 0-based and contiguous, got {indices}" -def test_pdf_image_captioning_on_the_fly(monkeypatch): - # Mock extract_pdf_images to yield one image on page 1 - def fake_extract_images(doc_or_path, **kwargs): - yield { - "image_bytes": b"fake_png_bytes", - "page": 1, - "width": 100, - "height": 100, - } - - monkeypatch.setattr(chunker, "extract_pdf_images", fake_extract_images) - - # Mock caption_image to return a custom caption - from app.rag import vision - monkeypatch.setattr(vision, "caption_image", lambda img_bytes, page=None: f"Captured image on page {page}") - - # Mock extract_pdf to return a text page - def fake_extract_pdf(filepath): - return [{"text": "Hello world on page 1", "page": 1, "chunk_type": "text"}] - - monkeypatch.setattr(chunker, "extract_pdf", fake_extract_pdf) - - # Mock fitz.open - class FakePage: - rect = type('Rect', (), {'width': 100, 'height': 100})() - def search_for(self, text): - return [] - - class FakePdf: - def __init__(self, *args, **kwargs): - pass - def __len__(self): - return 1 - def __getitem__(self, idx): - return FakePage() - def close(self): - pass - - import fitz - monkeypatch.setattr(fitz, "open", lambda *args, **kwargs: FakePdf()) - - # Run chunk_document - chunks = chunk_document("dummy.pdf") - - # The result should contain the text chunk and the image chunk - assert len(chunks) == 2 - assert chunks[0]["text"] == "Hello world on page 1" - assert chunks[0]["page"] == 1 - - assert chunks[1]["text"] == "Captured image on page 1" - assert chunks[1]["page"] == 1 - assert chunks[1]["is_image"] is True - assert chunks[1]["image_caption"] == "Captured image on page 1" - # Ensure image_bytes is not in the chunk dictionary (preventing memory leak) - assert "image_bytes" not in chunks[1] - -def test_pdf_multi_column_layout_parsing_order(monkeypatch): - """Ensure multi-column layout extracts text column-by-column rather than row-by-row.""" - class FakePage: - width = 600 - height = 800 - - def find_tables(self): - return [] - - def extract_words(self): - # Simulated 2-column layout with 2 sequential reading rows - return [ - # Left Column - Line 1 - {"text": "Left1", "x0": 50, "x1": 100, "top": 100, "bottom": 115}, - # Right Column - Line 1 - {"text": "Right1", "x0": 350, "x1": 400, "top": 100, "bottom": 115}, - # Left Column - Line 2 - {"text": "Left2", "x0": 50, "x1": 100, "top": 130, "bottom": 145}, - # Right Column - Line 2 - {"text": "Right2", "x0": 350, "x1": 400, "top": 130, "bottom": 145}, - ] - - class FakePdf: - pages = [FakePage()] - def __enter__(self): return self - def __exit__(self, exc_type, exc, traceback): return False - - fake_pdfplumber = types.SimpleNamespace(open=lambda _filepath: FakePdf()) - monkeypatch.setitem(sys.modules, "pdfplumber", fake_pdfplumber) - monkeypatch.setattr(chunker, "extract_pdf_images", lambda _filepath: []) - - chunks = chunk_document("multi_column.pdf") - - assert len(chunks) == 1 - text_lines = chunks[0]["text"].splitlines() - - # Correct reading order reads entire Column 1 downward, then moves to Column 2 - assert text_lines == ["Left1", "Left2", "Right1", "Right2"] diff --git a/backend/tests/test_document_upload_validation.py b/backend/tests/test_document_upload_validation.py index df8661cc..10bc252e 100644 --- a/backend/tests/test_document_upload_validation.py +++ b/backend/tests/test_document_upload_validation.py @@ -6,8 +6,7 @@ from pathlib import Path import pytest -from fastapi import UploadFile -from app.exceptions import ValidationException +from fastapi import HTTPException, UploadFile from pypdf import PdfWriter from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -55,11 +54,11 @@ def test_validate_upload_accepts_valid_pdf() -> None: def test_validate_upload_rejects_invalid_file_type() -> None: - with pytest.raises(ValidationException) as exc: + with pytest.raises(HTTPException) as exc: _run(documents.validate_upload(_upload_file("notes.exe", b"not a document"))) assert exc.value.status_code == 400 - assert "Only PDF" in exc.value.message + assert "Only PDF" in exc.value.detail def test_validate_upload_rejects_oversized_file_and_removes_temp_file( @@ -78,21 +77,21 @@ def tracking_tempfile(*args, **kwargs): monkeypatch.setattr(documents.settings, "MAX_UPLOAD_SIZE_MB", 0) monkeypatch.setattr(documents.tempfile, "NamedTemporaryFile", tracking_tempfile) - with pytest.raises(ValidationException) as exc: + with pytest.raises(HTTPException) as exc: _run(documents.validate_upload(_upload_file("too-large.pdf", _pdf_bytes()))) assert exc.value.status_code == 400 - assert exc.value.message == "File too large" + assert exc.value.detail == "File too large" assert created_paths assert all(not path.exists() for path in created_paths) def test_validate_upload_rejects_corrupted_pdf() -> None: - with pytest.raises(ValidationException) as exc: + with pytest.raises(HTTPException) as exc: _run(documents.validate_upload(_upload_file("broken.pdf", b"%PDF-1.4\nnot really a pdf"))) assert exc.value.status_code == 400 - assert exc.value.message == "Corrupted or invalid file" + assert exc.value.detail == "Corrupted or invalid file" @pytest.mark.parametrize( @@ -151,8 +150,6 @@ def __init__(self, value: str) -> None: first = _run( documents.upload_document( file=_upload_file("same-name.pdf", b"first"), - chunk_size=1000, - chunk_overlap=200, user=user, db=session, ) @@ -160,8 +157,6 @@ def __init__(self, value: str) -> None: second = _run( documents.upload_document( file=_upload_file("same-name.pdf", b"second"), - chunk_size=1000, - chunk_overlap=200, user=user, db=session, ) diff --git a/backend/tests/test_documents.py b/backend/tests/test_documents.py index 6fd78582..f66f8040 100644 --- a/backend/tests/test_documents.py +++ b/backend/tests/test_documents.py @@ -37,7 +37,7 @@ def test_upload_rejects_unsupported_extension_before_deep_validation(client, aut ) assert response.status_code == 400 - assert "not supported" in response.json()["error"]["message"] + assert "not supported" in response.json()["detail"] def test_rename_document_updates_original_name(client, auth_headers, ready_document, db_session): @@ -77,110 +77,6 @@ def test_rename_document_returns_404_for_missing_document(client, auth_headers): assert response.status_code == 404 -def test_update_document_preserves_existing_rename_behavior(client, auth_headers, ready_document, db_session): - """The new DocumentUpdate schema should still accept 'name' the same way - the old DocumentRename schema did — backward compatibility.""" - response = client.patch( - f"/api/v1/documents/{ready_document.id}", - headers=auth_headers, - json={"name": " renamed-report.pdf "}, - ) - - assert response.status_code == 200 - payload = response.json() - assert payload["id"] == ready_document.id - assert payload["original_name"] == "renamed-report.pdf" - - db_session.refresh(ready_document) - assert ready_document.original_name == "renamed-report.pdf" - assert ready_document.filename == "ready.txt" - - -def test_update_document_sets_summary(client, auth_headers, ready_document, db_session): - response = client.patch( - f"/api/v1/documents/{ready_document.id}", - headers=auth_headers, - json={"summary": "This is a test document for RAG evaluation."}, - ) - - assert response.status_code == 200 - payload = response.json() - assert payload["id"] == ready_document.id - assert payload["summary"] == "This is a test document for RAG evaluation." - - db_session.refresh(ready_document) - assert ready_document.summary == "This is a test document for RAG evaluation." - - -def test_update_document_sets_both_name_and_summary(client, auth_headers, ready_document, db_session): - response = client.patch( - f"/api/v1/documents/{ready_document.id}", - headers=auth_headers, - json={"name": "full-update.txt", "summary": "Both fields updated."}, - ) - - assert response.status_code == 200 - payload = response.json() - assert payload["original_name"] == "full-update.txt" - assert payload["summary"] == "Both fields updated." - - db_session.refresh(ready_document) - assert ready_document.original_name == "full-update.txt" - assert ready_document.summary == "Both fields updated." - - -def test_update_document_clears_summary_with_empty_string(client, auth_headers, ready_document, db_session): - # First set a summary - ready_document.summary = "Existing summary" - db_session.commit() - - # Then clear it with empty string - response = client.patch( - f"/api/v1/documents/{ready_document.id}", - headers=auth_headers, - json={"summary": ""}, - ) - - assert response.status_code == 200 - payload = response.json() - assert payload["summary"] is None - - db_session.refresh(ready_document) - assert ready_document.summary is None - - -def test_update_document_clears_summary_with_whitespace(client, auth_headers, ready_document, db_session): - ready_document.summary = "Existing summary" - db_session.commit() - - response = client.patch( - f"/api/v1/documents/{ready_document.id}", - headers=auth_headers, - json={"summary": " \t "}, - ) - - assert response.status_code == 200 - payload = response.json() - assert payload["summary"] is None - - db_session.refresh(ready_document) - assert ready_document.summary is None - - -def test_update_document_no_fields_does_nothing(client, auth_headers, ready_document, db_session): - """Sending an empty body (or body with no known fields) is a no-op.""" - response = client.patch( - f"/api/v1/documents/{ready_document.id}", - headers=auth_headers, - json={}, - ) - - assert response.status_code == 200 - db_session.refresh(ready_document) - assert ready_document.original_name == "ready.txt" - assert ready_document.summary is None - - def test_rename_document_returns_403_for_other_users_document(client, auth_headers, db_session, other_user): other_document = Document( user_id=other_user.id, diff --git a/backend/tests/test_exception_handling.py b/backend/tests/test_exception_handling.py deleted file mode 100644 index 007761b1..00000000 --- a/backend/tests/test_exception_handling.py +++ /dev/null @@ -1,209 +0,0 @@ -import asyncio -import json -from unittest.mock import Mock - -import pytest - -from fastapi import Request -from fastapi.exceptions import RequestValidationError -from slowapi.errors import RateLimitExceeded - -from app.exceptions import ( - AppException, - ConflictException, - ExternalServiceException, - ForbiddenException, - NotFoundException, - RateLimitException, - UnauthorizedException, - UnsafePromptException, - ValidationException, -) -from app.main import ( - app_exception_handler, - rate_limit_handler, - unhandled_exception_handler, - validation_exception_handler, -) - - -def _mock_request(request_id="test-123"): - request = Mock() - request.state = Mock(request_id=request_id) - return request - - -# ────────────────────────────────────────────── -# Exception class instantiation (uncovered lines) -# ────────────────────────────────────────────── - - -def test_app_exception_defaults(): - exc = AppException("CODE", "msg", 500) - assert exc.code == "CODE" - assert exc.message == "msg" - assert exc.status_code == 500 - assert exc.details == {} - - -def test_app_exception_with_details(): - exc = AppException("CODE", "msg", 400, {"key": "val"}) - assert exc.details == {"key": "val"} - - -def test_not_found_with_identifier(): - exc = NotFoundException("document", "42") - assert exc.code == "DOCUMENT_NOT_FOUND" - assert "42" in exc.message - assert exc.status_code == 404 - assert exc.details == {"document": "42"} - - -def test_not_found_without_identifier(): - exc = NotFoundException("user") - assert exc.code == "USER_NOT_FOUND" - assert exc.status_code == 404 - assert exc.details == {} - - -def test_unauthorized_default(): - exc = UnauthorizedException() - assert exc.code == "UNAUTHORIZED" - assert exc.status_code == 401 - - -def test_forbidden_default(): - exc = ForbiddenException() - assert exc.code == "FORBIDDEN" - assert exc.status_code == 403 - - -def test_conflict(): - exc = ConflictException("Username taken") - assert exc.code == "CONFLICT" - assert exc.status_code == 409 - assert exc.message == "Username taken" - - -def test_conflict_with_details(): - exc = ConflictException("Conflict", {"field": "username"}) - assert exc.details == {"field": "username"} - - -def test_validation(): - exc = ValidationException("bad input", {"field": "email"}) - assert exc.code == "VALIDATION_ERROR" - assert exc.status_code == 400 - assert exc.details == {"field": "email"} - - -def test_rate_limit(): - exc = RateLimitException() - assert exc.code == "RATE_LIMIT_EXCEEDED" - assert exc.status_code == 429 - assert "Rate limit" in exc.message - - -def test_external_service(): - exc = ExternalServiceException("openai", "key expired") - assert exc.code == "OPENAI_ERROR" - assert exc.status_code == 502 - assert exc.details == {"service": "openai"} - - -def test_external_service_default_message(): - exc = ExternalServiceException("stripe") - assert "stripe" in exc.message - - -def test_unsafe_prompt(): - exc = UnsafePromptException() - assert exc.code == "UNSAFE_PROMPT" - assert exc.status_code == 400 - assert "prohibited" in exc.message - - -def test_unsafe_prompt_custom_message(): - exc = UnsafePromptException("Custom block message") - assert exc.message == "Custom block message" - - -# ────────────────────────────────────────────── -# Exception handler response format (uncovered lines) -# ────────────────────────────────────────────── - - -def test_app_exception_handler(): - request = _mock_request() - exc = AppException("NOT_FOUND", "nope", 404, {"id": "1"}) - response = asyncio.run(app_exception_handler(request, exc)) - assert response.status_code == 404 - body = json.loads(response.body) - assert body == { - "error": { - "code": "NOT_FOUND", - "message": "nope", - "details": {"id": "1"}, - "request_id": "test-123", - } - } - - -def test_app_exception_handler_no_request_id(): - request = Mock() - request.state = Mock(spec_set=[]) - response = asyncio.run( - app_exception_handler(request, AppException("E", "e", 500)) - ) - body = json.loads(response.body) - assert body["error"]["request_id"] is None - - -def test_rate_limit_handler(): - request = _mock_request() - exc = RateLimitExceeded() - response = asyncio.run(rate_limit_handler(request, exc)) - assert response.status_code == 429 - body = json.loads(response.body) - assert body["error"]["code"] == "RATE_LIMIT_EXCEEDED" - assert body["error"]["request_id"] == "test-123" - - -def test_validation_exception_handler(): - request = _mock_request() - exc = RequestValidationError( - errors=[ - { - "loc": ("body", "email"), - "msg": "field required", - "type": "value_error", - } - ] - ) - response = asyncio.run(validation_exception_handler(request, exc)) - assert response.status_code == 422 - body = json.loads(response.body) - assert body["error"]["code"] == "VALIDATION_ERROR" - assert body["error"]["details"]["errors"][0]["field"] == "body -> email" - - -def test_unhandled_exception_handler(monkeypatch): - monkeypatch.setattr("app.main.settings.DEBUG", False) - request = _mock_request() - exc = ValueError("unexpected") - response = asyncio.run(unhandled_exception_handler(request, exc)) - assert response.status_code == 500 - body = json.loads(response.body) - assert body["error"]["code"] == "INTERNAL_ERROR" - assert body["error"]["message"] == "An unexpected error occurred" - - -def test_unhandled_exception_handler_debug_raises(monkeypatch): - monkeypatch.setattr("app.main.settings.DEBUG", True) - request = _mock_request() - exc = ValueError("unexpected") - try: - raise exc - except ValueError: - with pytest.raises(ValueError, match="unexpected"): - asyncio.run(unhandled_exception_handler(request, exc)) diff --git a/backend/tests/test_migrations.py b/backend/tests/test_migrations.py deleted file mode 100644 index 5f17c278..00000000 --- a/backend/tests/test_migrations.py +++ /dev/null @@ -1,408 +0,0 @@ -""" -Unit tests for database schema migrations (_migrate_schema). - -Each test creates a minimal in-memory SQLite database that intentionally -omits one or more columns (simulating an older schema), runs -_migrate_schema(), and then verifies the missing columns were added -without corrupting existing data or dropping any existing columns. -""" -import pytest -from sqlalchemy import create_engine, inspect, text -from sqlalchemy.orm import sessionmaker - - -# ── Helpers ─────────────────────────────────────────────────────────────────── - -def _make_engine(): - """Fresh in-memory SQLite engine — fully isolated per test.""" - return create_engine( - "sqlite:///:memory:", - connect_args={"check_same_thread": False}, - ) - - -def _columns(engine, table: str) -> set: - """Return the set of column names for a table.""" - return {c["name"] for c in inspect(engine).get_columns(table)} - - -def _run_migrate(engine): - """Patch app.database.engine and call _migrate_schema().""" - import app.database as db_module - from sqlalchemy import inspect as sa_inspect - - original_engine = db_module.engine - db_module.engine = engine - - # Also patch inspect so _migrate_schema uses our engine's inspector - original_inspect = db_module.inspect - db_module.inspect = lambda _: sa_inspect(engine) - - try: - db_module._migrate_schema() - finally: - db_module.engine = original_engine - db_module.inspect = original_inspect - - -def _create_minimal_users(engine): - """Create users table with only the original columns (no migration columns).""" - with engine.begin() as conn: - conn.execute(text(""" - CREATE TABLE users ( - id CHAR(36) PRIMARY KEY, - username VARCHAR(80) NOT NULL UNIQUE, - email VARCHAR(120) NOT NULL UNIQUE, - hashed_password VARCHAR(255) NOT NULL, - is_admin BOOLEAN DEFAULT FALSE, - created_at TIMESTAMP - ) - """)) - - -def _create_minimal_documents(engine): - """Create documents table with only the original columns.""" - with engine.begin() as conn: - conn.execute(text(""" - CREATE TABLE documents ( - id CHAR(36) PRIMARY KEY, - user_id CHAR(36) NOT NULL, - filename VARCHAR(255) NOT NULL, - original_name VARCHAR(255) NOT NULL, - file_size INTEGER DEFAULT 0, - page_count INTEGER DEFAULT 0, - chunk_count INTEGER DEFAULT 0, - status VARCHAR(20) DEFAULT 'pending', - error_message TEXT, - uploaded_at TIMESTAMP - ) - """)) - - -def _create_minimal_api_keys(engine): - """Create api_keys table with only the original columns.""" - with engine.begin() as conn: - conn.execute(text(""" - CREATE TABLE api_keys ( - id CHAR(36) PRIMARY KEY, - user_id CHAR(36) NOT NULL, - key_prefix VARCHAR(20) NOT NULL, - hashed_key VARCHAR(255) NOT NULL UNIQUE - ) - """)) - - -def _create_minimal_chat_messages(engine): - """Create chat_messages table without the feedback column.""" - with engine.begin() as conn: - conn.execute(text(""" - CREATE TABLE chat_messages ( - id CHAR(36) PRIMARY KEY, - user_id CHAR(36) NOT NULL, - document_id CHAR(36), - session_id CHAR(36), - role VARCHAR(20) NOT NULL, - content TEXT NOT NULL, - sources_json TEXT, - created_at TIMESTAMP - ) - """)) - - -def _setup_all_minimal_tables(engine): - """Create all tables with older schemas so the migration script doesn't crash.""" - _create_minimal_users(engine) - _create_minimal_documents(engine) - _create_minimal_api_keys(engine) - _create_minimal_chat_messages(engine) - - -# ── users migrations ────────────────────────────────────────────────────────── - -def test_migrate_adds_hf_token_to_users(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - assert "hf_token" not in _columns(engine, "users") - _run_migrate(engine) - assert "hf_token" in _columns(engine, "users") - - -def test_migrate_adds_role_to_users(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - assert "role" not in _columns(engine, "users") - _run_migrate(engine) - assert "role" in _columns(engine, "users") - - -def test_migrate_adds_google_refresh_token_to_users(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - assert "google_refresh_token" not in _columns(engine, "users") - _run_migrate(engine) - assert "google_refresh_token" in _columns(engine, "users") - - -def test_migrate_adds_last_login_to_users(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - _run_migrate(engine) - assert "last_login" in _columns(engine, "users") - - -def test_migrate_adds_is_verified_to_users(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - assert "is_verified" not in _columns(engine, "users") - _run_migrate(engine) - assert "is_verified" in _columns(engine, "users") - - -def test_migrate_adds_verification_token_hash_to_users(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - _run_migrate(engine) - assert "verification_token_hash" in _columns(engine, "users") - - -def test_migrate_adds_verification_token_created_at_to_users(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - _run_migrate(engine) - assert "verification_token_created_at" in _columns(engine, "users") - - -def test_migrate_preserves_existing_users_columns(): - """Migration must never drop or rename any original users column.""" - engine = _make_engine() - _setup_all_minimal_tables(engine) - original = _columns(engine, "users") - _run_migrate(engine) - after = _columns(engine, "users") - assert original.issubset(after), f"Columns removed: {original - after}" - - -def test_migrate_preserves_existing_user_data(): - """Rows inserted before migration must still be readable afterwards.""" - engine = _make_engine() - _setup_all_minimal_tables(engine) - with engine.begin() as conn: - conn.execute(text( - "INSERT INTO users (id, username, email, hashed_password) " - "VALUES ('u1', 'alice', 'alice@example.com', 'hash')" - )) - _run_migrate(engine) - with engine.connect() as conn: - row = conn.execute( - text("SELECT username, email FROM users WHERE id = 'u1'") - ).fetchone() - assert row is not None - assert row[0] == "alice" - assert row[1] == "alice@example.com" - - -# ── documents migrations ────────────────────────────────────────────────────── - -def test_migrate_adds_is_deleted_to_documents(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - assert "is_deleted" not in _columns(engine, "documents") - _run_migrate(engine) - assert "is_deleted" in _columns(engine, "documents") - - -def test_migrate_adds_summary_to_documents(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - assert "summary" not in _columns(engine, "documents") - _run_migrate(engine) - assert "summary" in _columns(engine, "documents") - - -def test_migrate_adds_chunk_size_and_overlap_to_documents(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - _run_migrate(engine) - cols = _columns(engine, "documents") - assert "chunk_size" in cols - assert "chunk_overlap" in cols - - -def test_migrate_adds_processing_progress_to_documents(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - _run_migrate(engine) - assert "processing_progress" in _columns(engine, "documents") - - -def test_migrate_adds_processing_stage_to_documents(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - _run_migrate(engine) - assert "processing_stage" in _columns(engine, "documents") - - -def test_migrate_adds_extracted_urls_to_documents(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - assert "extracted_urls" not in _columns(engine, "documents") - _run_migrate(engine) - assert "extracted_urls" in _columns(engine, "documents") - - -def test_migrate_adds_completed_at_to_documents(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - _run_migrate(engine) - assert "completed_at" in _columns(engine, "documents") - - -def test_migrate_adds_drive_columns_to_documents(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - _run_migrate(engine) - cols = _columns(engine, "documents") - assert "drive_file_id" in cols - assert "drive_folder_id" in cols - assert "drive_synced_at" in cols - - -def test_migrate_preserves_existing_documents_columns(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - original = _columns(engine, "documents") - _run_migrate(engine) - after = _columns(engine, "documents") - assert original.issubset(after), f"Columns removed: {original - after}" - - -def test_migrate_preserves_existing_document_data(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - with engine.begin() as conn: - conn.execute(text( - "INSERT INTO documents " - "(id, user_id, filename, original_name, file_size, status) " - "VALUES ('d1', 'u1', 'file.pdf', 'file.pdf', 1024, 'ready')" - )) - _run_migrate(engine) - with engine.connect() as conn: - row = conn.execute( - text("SELECT original_name, status FROM documents WHERE id = 'd1'") - ).fetchone() - assert row is not None - assert row[0] == "file.pdf" - assert row[1] == "ready" - - -# ── api_keys migrations ─────────────────────────────────────────────────────── - -def test_migrate_adds_name_to_api_keys(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - assert "name" not in _columns(engine, "api_keys") - _run_migrate(engine) - assert "name" in _columns(engine, "api_keys") - - -def test_migrate_adds_is_active_to_api_keys(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - assert "is_active" not in _columns(engine, "api_keys") - _run_migrate(engine) - assert "is_active" in _columns(engine, "api_keys") - - -def test_migrate_adds_last_used_at_to_api_keys(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - _run_migrate(engine) - assert "last_used_at" in _columns(engine, "api_keys") - - -def test_migrate_preserves_existing_api_keys_columns(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - original = _columns(engine, "api_keys") - _run_migrate(engine) - after = _columns(engine, "api_keys") - assert original.issubset(after), f"Columns removed: {original - after}" - - -# ── chat_messages migrations ────────────────────────────────────────────────── - -def test_migrate_adds_feedback_to_chat_messages(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - assert "feedback" not in _columns(engine, "chat_messages") - _run_migrate(engine) - assert "feedback" in _columns(engine, "chat_messages") - - -def test_migrate_preserves_existing_chat_messages_columns(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - original = _columns(engine, "chat_messages") - _run_migrate(engine) - after = _columns(engine, "chat_messages") - assert original.issubset(after), f"Columns removed: {original - after}" - - -# ── Idempotency ─────────────────────────────────────────────────────────────── - -def test_migrate_is_idempotent_users(): - """Running _migrate_schema() twice must not raise or duplicate columns.""" - engine = _make_engine() - _setup_all_minimal_tables(engine) - _run_migrate(engine) - cols_after_first = _columns(engine, "users") - _run_migrate(engine) - cols_after_second = _columns(engine, "users") - assert cols_after_first == cols_after_second - - -def test_migrate_is_idempotent_documents(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - _run_migrate(engine) - cols_after_first = _columns(engine, "documents") - _run_migrate(engine) - cols_after_second = _columns(engine, "documents") - assert cols_after_first == cols_after_second - - -def test_migrate_is_idempotent_api_keys(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - _run_migrate(engine) - cols_after_first = _columns(engine, "api_keys") - _run_migrate(engine) - cols_after_second = _columns(engine, "api_keys") - assert cols_after_first == cols_after_second - - -def test_migrate_is_idempotent_chat_messages(): - engine = _make_engine() - _setup_all_minimal_tables(engine) - _run_migrate(engine) - cols_after_first = _columns(engine, "chat_messages") - _run_migrate(engine) - cols_after_second = _columns(engine, "chat_messages") - assert cols_after_first == cols_after_second - - -# ── Full schema (already migrated) ─────────────────────────────────────────── - -def test_migrate_on_fully_migrated_schema_is_safe(): - """If all columns already exist, _migrate_schema() must complete without error.""" - engine = _make_engine() - _setup_all_minimal_tables(engine) - # First pass adds all missing columns - _run_migrate(engine) - # Second pass should run safely as a complete no-op - _run_migrate(engine) - # Verify nothing was lost - assert "hf_token" in _columns(engine, "users") - assert "extracted_urls" in _columns(engine, "documents") - assert "is_active" in _columns(engine, "api_keys") - assert "feedback" in _columns(engine, "chat_messages") diff --git a/backend/tests/test_multi_document_chat.py b/backend/tests/test_multi_document_chat.py deleted file mode 100644 index cce84d45..00000000 --- a/backend/tests/test_multi_document_chat.py +++ /dev/null @@ -1,164 +0,0 @@ -from unittest.mock import MagicMock - -from app.rag import retriever -from app.models import Document - - -# ── Retrieval: document_ids reaches both vector and BM25 (dev's direct-call retrieve) ── - -def _mock_db(monkeypatch, doc_rows): - mock_db = MagicMock() - mock_db.__enter__.return_value = mock_db - mock_query = MagicMock() - mock_db.query.return_value = mock_query - mock_query.filter.return_value.all.return_value = doc_rows - monkeypatch.setattr("app.database.SessionLocal", lambda: mock_db) - - -def test_retrieve_forwards_document_ids_to_vector_and_bm25(monkeypatch): - _mock_db(monkeypatch, [("doc-a",), ("doc-b",)]) - - seen = {"vector": "unset", "bm25": "unset"} - - monkeypatch.setattr(retriever, "transform_query", lambda _q: ["q"]) - monkeypatch.setattr(retriever, "embed_query", lambda q: f"embedding:{q}") - monkeypatch.setattr(retriever, "get_reranker", lambda: None) - - def fake_query_chunks(query_embedding, user_id, document_id=None, document_ids=None, top_k=10): - seen["vector"] = document_ids - return [{"id": "v1", "text": "vec", "filename": "a.pdf", "page": 1, "score": 0.5}] - - def fake_query_bm25(query, user_id, document_id=None, document_ids=None, top_k=10): - seen["bm25"] = document_ids - return [{"id": "b1", "text": "bm", "filename": "b.pdf", "page": 1, "score": 0.5}] - - monkeypatch.setattr(retriever, "query_chunks", fake_query_chunks) - monkeypatch.setattr("app.rag.bm25.query_bm25", fake_query_bm25) - - retriever.retrieve("question", user_id="user-1", document_ids=["doc-a", "doc-b"]) - - assert seen["vector"] == ["doc-a", "doc-b"] - # bm25 only runs when hybrid search is enabled; if it ran, it must have received the ids - if seen["bm25"] != "unset": - assert seen["bm25"] == ["doc-a", "doc-b"] - - -def test_retrieve_single_document_leaves_document_ids_none(monkeypatch): - _mock_db(monkeypatch, [("doc-a",)]) - - seen = {"vector_id": "unset", "vector_ids": "unset"} - - monkeypatch.setattr(retriever, "transform_query", lambda _q: ["q"]) - monkeypatch.setattr(retriever, "embed_query", lambda q: f"embedding:{q}") - monkeypatch.setattr(retriever, "get_reranker", lambda: None) - - def fake_query_chunks(query_embedding, user_id, document_id=None, document_ids=None, top_k=10): - seen["vector_id"] = document_id - seen["vector_ids"] = document_ids - return [{"id": "v1", "text": "vec", "filename": "a.pdf", "page": 1, "score": 0.5}] - - monkeypatch.setattr(retriever, "query_chunks", fake_query_chunks) - - retriever.retrieve("question", user_id="user-1", document_id="doc-a") - - assert seen["vector_id"] == "doc-a" - assert seen["vector_ids"] is None - - -# ── Prompt: comparison guidance only when more than one document ── - -def test_comparison_guidance_present_only_for_multiple_documents(monkeypatch): - from app.rag import agent - from app.rag.prompts import MULTI_DOC_COMPARISON_GUIDANCE - - captured = {} - - class FakeLLM: - def __init__(self, *a, **k): - pass - - def capture_prompt(llm, tools, prompt): - captured["template"] = prompt.template - return "agent" - - monkeypatch.setattr(agent, "get_llm_client", lambda hf_token=None: FakeLLM()) - monkeypatch.setattr(agent, "create_react_agent", capture_prompt) - monkeypatch.setattr(agent, "AgentExecutor", lambda **kwargs: kwargs) - - agent.get_agent_executor(user_id="user-1", document_ids=["doc-a", "doc-b"]) - assert MULTI_DOC_COMPARISON_GUIDANCE.strip() in captured["template"] - - captured.clear() - agent.get_agent_executor(user_id="user-1", document_id="doc-a") - assert MULTI_DOC_COMPARISON_GUIDANCE.strip() not in captured["template"] - - -# ── Route guard: ownership + readiness for document_ids ── - -def test_chat_ask_multi_doc_success(client, auth_headers, ready_document, db_session, user, monkeypatch): - second = Document( - user_id=user.id, - filename="second.txt", - original_name="second.txt", - file_size=128, - status="ready", - ) - db_session.add(second) - db_session.commit() - db_session.refresh(second) - - monkeypatch.setattr( - "app.routes.chat.generate_answer", - lambda question, user_id, document_id=None, document_ids=None, **kwargs: { - "answer": "Across both docs", - "sources": [], - }, - ) - - response = client.post( - "/api/v1/chat/ask", - headers=auth_headers, - json={"question": "Compare them", "document_ids": [ready_document.id, second.id]}, - ) - - assert response.status_code == 200 - assert response.json()["answer"] == "Across both docs" - - -def test_chat_ask_multi_doc_rejects_missing_document(client, auth_headers, ready_document): - response = client.post( - "/api/v1/chat/ask", - headers=auth_headers, - json={"question": "Compare", "document_ids": [ready_document.id, "missing-doc-id"]}, - ) - assert response.status_code == 404 - - -def test_chat_ask_multi_doc_rejects_not_ready_document(client, auth_headers, ready_document, pending_document): - response = client.post( - "/api/v1/chat/ask", - headers=auth_headers, - json={"question": "Compare", "document_ids": [ready_document.id, pending_document.id]}, - ) - assert response.status_code == 400 - - -def test_chat_ask_multi_doc_rejects_other_users_document(client, auth_headers, ready_document, db_session, other_user): - other_doc = Document( - user_id=other_user.id, - filename="other.txt", - original_name="other.txt", - file_size=64, - status="ready", - ) - db_session.add(other_doc) - db_session.commit() - db_session.refresh(other_doc) - - response = client.post( - "/api/v1/chat/ask", - headers=auth_headers, - json={"question": "Compare", "document_ids": [ready_document.id, other_doc.id]}, - ) - # not owned -> treated as missing - assert response.status_code == 404 \ No newline at end of file diff --git a/backend/tests/test_ocr.py b/backend/tests/test_ocr.py deleted file mode 100644 index 59892def..00000000 --- a/backend/tests/test_ocr.py +++ /dev/null @@ -1,259 +0,0 @@ -""" -Tests for OCR fallback — issue #282. - -All external I/O (fitz, pytesseract, easyocr, Pillow) is mocked so the -suite runs without Tesseract or any GPU dependency installed. -""" -import types -from unittest.mock import MagicMock, patch, PropertyMock - -import pytest - - -# ── helpers ────────────────────────────────────────────────────────────────── - -def _make_fitz_page(text: str = "") -> MagicMock: - """Return a mock fitz.Page whose get_text() returns *text*.""" - page = MagicMock() - page.get_text.return_value = text - pix = MagicMock() - pix.tobytes.return_value = b"PNG_BYTES" - page.get_pixmap.return_value = pix - return page - - -def _make_fitz_doc(pages_text: list[str]): - """Return a mock fitz document iterating over mock pages.""" - pages = [_make_fitz_page(t) for t in pages_text] - doc = MagicMock() - doc.__iter__ = MagicMock(return_value=iter(pages)) - doc.__len__ = MagicMock(return_value=len(pages)) - doc.__enter__ = MagicMock(return_value=doc) - doc.__exit__ = MagicMock(return_value=False) - return doc, pages - - -# ── _page_is_image_only ─────────────────────────────────────────────────────── - -class TestPageIsImageOnly: - def test_empty_page_is_image_only(self): - from app.rag.ocr import _page_is_image_only - page = _make_fitz_page("") - assert _page_is_image_only(page) is True - - def test_sparse_page_is_image_only(self): - from app.rag.ocr import _page_is_image_only - page = _make_fitz_page("hi") - assert _page_is_image_only(page) is True - - def test_page_with_enough_text_is_not_image_only(self): - from app.rag.ocr import _page_is_image_only - page = _make_fitz_page("A" * 50) - assert _page_is_image_only(page) is False - - def test_boundary_exactly_at_min_chars(self): - from app.rag.ocr import _page_is_image_only, MIN_TEXT_CHARS - page = _make_fitz_page("A" * MIN_TEXT_CHARS) - assert _page_is_image_only(page) is False - - def test_one_below_boundary_is_image_only(self): - from app.rag.ocr import _page_is_image_only, MIN_TEXT_CHARS - page = _make_fitz_page("A" * (MIN_TEXT_CHARS - 1)) - assert _page_is_image_only(page) is True - - -# ── _render_page_to_image ───────────────────────────────────────────────────── - -class TestRenderPageToImage: - def test_returns_png_bytes(self): - from app.rag.ocr import _render_page_to_image - page = _make_fitz_page() - result = _render_page_to_image(page, dpi=72) - assert result == b"PNG_BYTES" - page.get_pixmap.assert_called_once() - - -# ── _ocr_with_tesseract ─────────────────────────────────────────────────────── - -class TestOcrWithTesseract: - def test_returns_extracted_text(self): - from app.rag.ocr import _ocr_with_tesseract - - mock_pytesseract = types.ModuleType("pytesseract") - mock_pytesseract.image_to_string = MagicMock(return_value=" Hello OCR ") - - mock_pil_image = MagicMock() - mock_pil_module = types.ModuleType("PIL") - mock_pil_image_class = MagicMock(return_value=mock_pil_image) - mock_pil_module.Image = MagicMock() - mock_pil_module.Image.open = MagicMock(return_value=mock_pil_image) - - with patch.dict( - "sys.modules", - {"pytesseract": mock_pytesseract, "PIL": mock_pil_module, "PIL.Image": mock_pil_module.Image}, - ): - result = _ocr_with_tesseract(b"PNG_BYTES") - - assert result == "Hello OCR" - - def test_raises_import_error_when_tesseract_missing(self): - from app.rag.ocr import _ocr_with_tesseract - with patch.dict("sys.modules", {"pytesseract": None, "PIL": None, "PIL.Image": None}): - with pytest.raises(ImportError, match="pytesseract"): - _ocr_with_tesseract(b"PNG_BYTES") - - -# ── ocr_page ───────────────────────────────────────────────────────────────── - -class TestOcrPage: - def test_uses_tesseract_by_default(self, monkeypatch): - import app.rag.ocr as ocr_module - monkeypatch.setattr(ocr_module, "OCR_BACKEND", "tesseract") - monkeypatch.setattr(ocr_module, "_render_page_to_image", lambda page, dpi: b"PNG") - monkeypatch.setattr(ocr_module, "_ocr_with_tesseract", lambda b: "tesseract text") - - page = _make_fitz_page() - result = ocr_module.ocr_page(page) - assert result == "tesseract text" - - def test_uses_easyocr_when_configured(self, monkeypatch): - import app.rag.ocr as ocr_module - monkeypatch.setattr(ocr_module, "OCR_BACKEND", "easyocr") - monkeypatch.setattr(ocr_module, "_render_page_to_image", lambda page, dpi: b"PNG") - monkeypatch.setattr(ocr_module, "_ocr_with_easyocr", lambda b: "easyocr text") - - page = _make_fitz_page() - result = ocr_module.ocr_page(page) - assert result == "easyocr text" - - -# ── extract_pdf_with_ocr ───────────────────────────────────────────────────── - -class TestExtractPdfWithOcr: - def test_native_text_pages_skip_ocr(self, monkeypatch, tmp_path): - import app.rag.ocr as ocr_module - - rich_text = "A" * 100 - doc, pages = _make_fitz_doc([rich_text]) - - monkeypatch.setattr("fitz.open", lambda path: doc) - ocr_called = [] - monkeypatch.setattr(ocr_module, "ocr_page", lambda page, dpi=200: ocr_called.append(1) or "") - - result = ocr_module.extract_pdf_with_ocr(str(tmp_path / "test.pdf")) - - assert len(result) == 1 - assert result[0]["ocr"] is False - assert result[0]["text"] == rich_text.strip() - assert ocr_called == [] - - def test_image_only_pages_trigger_ocr(self, monkeypatch, tmp_path): - import app.rag.ocr as ocr_module - - doc, _ = _make_fitz_doc([""]) # empty page - monkeypatch.setattr("fitz.open", lambda path: doc) - monkeypatch.setattr(ocr_module, "ocr_page", lambda page, dpi=200: "Scanned text here") - - result = ocr_module.extract_pdf_with_ocr(str(tmp_path / "scan.pdf")) - - assert len(result) == 1 - assert result[0]["ocr"] is True - assert result[0]["text"] == "Scanned text here" - assert result[0]["page"] == 1 - - def test_mixed_pages_handled_correctly(self, monkeypatch, tmp_path): - import app.rag.ocr as ocr_module - - rich = "B" * 100 - doc, _ = _make_fitz_doc([rich, ""]) - monkeypatch.setattr("fitz.open", lambda path: doc) - monkeypatch.setattr(ocr_module, "ocr_page", lambda page, dpi=200: "OCR result") - - result = ocr_module.extract_pdf_with_ocr(str(tmp_path / "mixed.pdf")) - - assert len(result) == 2 - assert result[0]["ocr"] is False - assert result[0]["page"] == 1 - assert result[1]["ocr"] is True - assert result[1]["page"] == 2 - - def test_ocr_returning_empty_skips_page(self, monkeypatch, tmp_path): - import app.rag.ocr as ocr_module - - doc, _ = _make_fitz_doc([""]) - monkeypatch.setattr("fitz.open", lambda path: doc) - monkeypatch.setattr(ocr_module, "ocr_page", lambda page, dpi=200: "") - - result = ocr_module.extract_pdf_with_ocr(str(tmp_path / "blank.pdf")) - assert result == [] - - def test_ocr_import_error_skips_page_gracefully(self, monkeypatch, tmp_path): - import app.rag.ocr as ocr_module - - doc, _ = _make_fitz_doc([""]) - monkeypatch.setattr("fitz.open", lambda path: doc) - monkeypatch.setattr( - ocr_module, "ocr_page", MagicMock(side_effect=ImportError("no tesseract")) - ) - - result = ocr_module.extract_pdf_with_ocr(str(tmp_path / "fail.pdf")) - assert result == [] - - def test_ocr_exception_skips_page_gracefully(self, monkeypatch, tmp_path): - import app.rag.ocr as ocr_module - - doc, _ = _make_fitz_doc([""]) - monkeypatch.setattr("fitz.open", lambda path: doc) - monkeypatch.setattr( - ocr_module, "ocr_page", MagicMock(side_effect=RuntimeError("segfault")) - ) - - result = ocr_module.extract_pdf_with_ocr(str(tmp_path / "crash.pdf")) - assert result == [] - - -# ── extract_pdf fallback chain (chunker integration) ───────────────────────── - -class TestExtractPdfOcrFallback: - def test_ocr_called_when_all_extractors_return_empty(self, monkeypatch): - import app.rag.chunker as chunker_module - import app.rag.ocr as ocr_module - - monkeypatch.setattr( - chunker_module, "extract_pdf_with_unstructured", - MagicMock(side_effect=Exception("unavailable")), - ) - monkeypatch.setattr( - chunker_module, "extract_pdf_with_tables", - MagicMock(side_effect=Exception("unavailable")), - ) - monkeypatch.setattr( - chunker_module, "extract_pdf_with_pymupdf", - MagicMock(return_value=[]), - ) - monkeypatch.setattr( - ocr_module, "extract_pdf_with_ocr", - MagicMock(return_value=[{"text": "OCR text", "page": 1, "chunk_type": "text", "ocr": True}]), - ) - - result = chunker_module.extract_pdf("dummy.pdf") - - assert len(result) == 1 - assert result[0]["ocr"] is True - assert result[0]["text"] == "OCR text" - - def test_ocr_not_called_when_extractor_succeeds(self, monkeypatch): - import app.rag.chunker as chunker_module - import app.rag.ocr as ocr_module - - monkeypatch.setattr( - chunker_module, "extract_pdf_with_unstructured", - MagicMock(return_value=[{"text": "Native text", "page": 1, "chunk_type": "text"}]), - ) - ocr_spy = MagicMock(return_value=[]) - monkeypatch.setattr(ocr_module, "extract_pdf_with_ocr", ocr_spy) - - result = chunker_module.extract_pdf("dummy.pdf") - - ocr_spy.assert_not_called() - assert result[0]["text"] == "Native text" diff --git a/backend/tests/test_profile.py b/backend/tests/test_profile.py deleted file mode 100644 index f2884517..00000000 --- a/backend/tests/test_profile.py +++ /dev/null @@ -1,28 +0,0 @@ -def test_profile_update_duplicate_username(client, auth_headers, other_user): - response = client.put( - "/profile/", - json={"username": "other"}, - headers=auth_headers, - ) - assert response.status_code == 400 - assert response.json()["error"]["message"] == "Username already exists" - - -def test_profile_update_keep_username(client, auth_headers, user): - response = client.put( - "/profile/", - json={"username": "tester"}, - headers=auth_headers, - ) - assert response.status_code == 200 - assert response.json()["username"] == "tester" - - -def test_profile_update_unique_username(client, auth_headers, user): - response = client.put( - "/profile/", - json={"username": "tester_new"}, - headers=auth_headers, - ) - assert response.status_code == 200 - assert response.json()["username"] == "tester_new" diff --git a/backend/tests/test_rag_tools.py b/backend/tests/test_rag_tools.py index 9726783c..30bbc9fa 100644 --- a/backend/tests/test_rag_tools.py +++ b/backend/tests/test_rag_tools.py @@ -154,7 +154,7 @@ def test_pdf_search_tool_formats_chunks_and_graph_context(monkeypatch): retrieve_calls = [] graph_calls = [] - def fake_retrieve(query, user_id, document_id=None, document_ids=None, top_k=None): + def fake_retrieve(query, user_id, document_id=None, top_k=None): retrieve_calls.append((query, user_id, document_id)) return chunks diff --git a/backend/tests/test_rate_limit.py b/backend/tests/test_rate_limit.py index 994f1787..5aae4f37 100644 --- a/backend/tests/test_rate_limit.py +++ b/backend/tests/test_rate_limit.py @@ -1,20 +1,8 @@ -""" -Unit tests for rate limiting middleware and identifier resolution (#445). - -Verifies key function fallback resolutions (User ID vs IP Address), route -attribute assignments, and that the global handler intercepts limit breaches -to return a 429 status code. -""" from types import SimpleNamespace -from uuid import uuid4 -import pytest -from fastapi.testclient import TestClient -from slowapi.errors import RateLimitExceeded from app.auth import create_access_token from app.rate_limit import CHAT_QUERY_RATE_LIMIT, rate_limit_key_func from app.routes.chat import ask_question, ask_question_stream -from app.main import app class DummyRequest: @@ -23,8 +11,6 @@ def __init__(self, headers=None): self.client = SimpleNamespace(host="203.0.113.10") -# ── Your Original Key Resolution & Route Tests ─────────────────────────────── - def test_rate_limit_key_prefers_authenticated_user_id(): token = create_access_token("user-123") @@ -45,74 +31,3 @@ def test_chat_endpoints_use_required_rate_limit(): assert CHAT_QUERY_RATE_LIMIT == "15/minute" assert ask_question.__rate_limits__ == [CHAT_QUERY_RATE_LIMIT] assert ask_question_stream.__rate_limits__ == [CHAT_QUERY_RATE_LIMIT] - - -# ── Middleware 429 Response Verification ────────────────────────────────────── - -def test_rate_limit_handler_returns_429(client: TestClient): - """ - Verify that hitting an endpoint that triggers a RateLimitExceeded exception - correctly triggers the global handler, returning a 429 status code and the - exact JSON error layout specified in app/main.py. - """ - from fastapi import APIRouter - test_router = APIRouter() - @test_router.get("/api/v1/test-rate-limiting-trigger-429") - def trigger_rate_limit(): - raise RateLimitExceeded("Too Many Requests") - - app.include_router(test_router) - # Move the newly added route to the top so it takes precedence over the SPA catch-all - new_route = app.router.routes.pop() - app.router.routes.insert(0, new_route) - - response = client.get("/api/v1/test-rate-limiting-trigger-429") - - # Verify the 429 status code requirement - assert response.status_code == 429 - - # Verify the specific JSON payload structure from app/main.py - json_data = response.json() - assert "error" in json_data - assert json_data["error"]["code"] == "RATE_LIMIT_EXCEEDED" - assert "Rate limit exceeded. Please try again later." in json_data["error"]["message"] - assert "request_id" in json_data["error"] - assert isinstance(json_data["error"]["details"], dict) - - -# ── Manual /chat/ws Rate Limit Enforcement ────────────────────────────────── - -def test_chat_ws_rate_limit_allows_up_to_configured_threshold(): - """check_chat_ws_rate_limit must allow exactly CHAT_QUERY_RATE_LIMIT hits.""" - from app.rate_limit import CHAT_QUERY_RATE_LIMIT, check_chat_ws_rate_limit - - limit = int(CHAT_QUERY_RATE_LIMIT.split("/")[0]) - user_id = f"ws-rate-limit-{uuid4()}" - - for _ in range(limit): - assert check_chat_ws_rate_limit(user_id) is True - - -def test_chat_ws_rate_limit_blocks_after_threshold_exceeded(): - """The hit beyond CHAT_QUERY_RATE_LIMIT must be rejected — this is the - bypass from #639, where /chat/ws had no rate limiting at all.""" - from app.rate_limit import CHAT_QUERY_RATE_LIMIT, check_chat_ws_rate_limit - - limit = int(CHAT_QUERY_RATE_LIMIT.split("/")[0]) - user_id = f"ws-rate-limit-{uuid4()}" - - for _ in range(limit): - check_chat_ws_rate_limit(user_id) - - assert check_chat_ws_rate_limit(user_id) is False - - -def test_chat_ws_rate_limit_is_scoped_per_user(): - """One user's usage must not consume another user's rate-limit budget.""" - from app.rate_limit import check_chat_ws_rate_limit - - user_a = f"ws-rate-limit-{uuid4()}" - user_b = f"ws-rate-limit-{uuid4()}" - - assert check_chat_ws_rate_limit(user_a) is True - assert check_chat_ws_rate_limit(user_b) is True \ No newline at end of file diff --git a/backend/tests/test_retriever.py b/backend/tests/test_retriever.py index 46306e92..6045dde4 100644 --- a/backend/tests/test_retriever.py +++ b/backend/tests/test_retriever.py @@ -24,21 +24,13 @@ def test_transform_query_includes_original_and_dedupes(monkeypatch): def test_retrieve_fans_out_transformed_queries_and_merges_duplicates(monkeypatch): - from unittest.mock import MagicMock searched_queries = [] - mock_db = MagicMock() - mock_db.__enter__.return_value = mock_db - mock_query = MagicMock() - mock_db.query.return_value = mock_query - mock_query.filter.return_value.all.return_value = [("doc-1",)] - monkeypatch.setattr("app.database.SessionLocal", lambda: mock_db) - monkeypatch.setattr(retriever, "transform_query", lambda _query: ["taxes", "healthcare"]) monkeypatch.setattr(retriever, "embed_query", lambda query: f"embedding:{query}") monkeypatch.setattr(retriever, "get_reranker", lambda: None) - def fake_query_chunks(query_embedding, user_id, document_id=None, document_ids=None, top_k=10): + def fake_query_chunks(query_embedding, user_id, document_id=None, top_k=10): searched_queries.append(query_embedding) if query_embedding == "embedding:taxes": return [ @@ -81,33 +73,5 @@ def fake_query_chunks(query_embedding, user_id, document_id=None, document_ids=N assert searched_queries == ["embedding:taxes", "embedding:healthcare"] assert [chunk["id"] for chunk in chunks] == ["shared", "taxes", "healthcare"] - assert chunks[0]["score"] > 0 # RRF score, not raw similarity - assert chunks[0]["id"] == "shared" # highest RRF score — appears in both query results + assert chunks[0]["score"] == 1.0 assert chunks[0]["confidence"] == 100.0 - - -def test_retrieve_includes_active_but_excludes_deleted(monkeypatch): - from unittest.mock import MagicMock - - mock_db = MagicMock() - mock_db.__enter__.return_value = mock_db - mock_query = MagicMock() - mock_db.query.return_value = mock_query - mock_query.filter.return_value.all.return_value = [("doc-active",)] - monkeypatch.setattr("app.database.SessionLocal", lambda: mock_db) - - monkeypatch.setattr(retriever, "transform_query", lambda _query: ["test"]) - monkeypatch.setattr(retriever, "embed_query", lambda query: f"embedding:{query}") - monkeypatch.setattr(retriever, "get_reranker", lambda: None) - - captured_ids = [] - def fake_query_chunks(query_embedding, user_id, document_id=None, document_ids=None, top_k=10): - captured_ids.append(document_ids) - return [{"id": "chunk-1", "text": "sample text", "filename": "policy.pdf", "page": 1, "score": 0.5}] - - monkeypatch.setattr(retriever, "query_chunks", fake_query_chunks) - - retriever.retrieve("hello", user_id="user-1") - - assert captured_ids == [None] # retrieve() passes document_id=None, not document_ids - diff --git a/backend/tests/test_share.py b/backend/tests/test_share.py index d8c23c66..f906c9b4 100644 --- a/backend/tests/test_share.py +++ b/backend/tests/test_share.py @@ -17,7 +17,7 @@ def test_share_link_unauthorized_for_other_users_message(client, auth_headers, o ) assert response.status_code == 404 - assert response.json()["error"]["message"] == "Message not found" + assert response.json()["detail"] == "Message not found" def test_cannot_share_user_message(client, auth_headers, user_message): @@ -27,14 +27,14 @@ def test_cannot_share_user_message(client, auth_headers, user_message): ) assert response.status_code == 400 - assert response.json()["error"]["message"] == "Only assistant messages can be shared" + assert response.json()["detail"] == "Only assistant messages can be shared" def test_public_fetch_fails_before_share(client, assistant_message): response = client.get(f"/api/v1/chat/share/{assistant_message.id}") assert response.status_code == 404 - assert response.json()["error"]["message"] == "Shared Answer not found" + assert response.json()["detail"] == "Shared answer not found" def test_public_fetch_shared_answer_success_after_share(client, auth_headers, assistant_message): @@ -58,4 +58,4 @@ def test_missing_message_returns_404(client): response = client.get("/api/v1/chat/share/missing-message-id") assert response.status_code == 404 - assert response.json()["error"]["message"] == "Shared Answer not found" + assert response.json()["detail"] == "Shared answer not found" diff --git a/backend/tests/test_tracing.py b/backend/tests/test_tracing.py deleted file mode 100644 index cae87444..00000000 --- a/backend/tests/test_tracing.py +++ /dev/null @@ -1,54 +0,0 @@ -import logging - -from app.rag import tracing - - -def test_trace_function_uses_metadata_factory(monkeypatch): - captured = {} - - def fake_trace_call(name, fn, *args, run_type, metadata, **kwargs): - captured.update(name=name, run_type=run_type, metadata=metadata) - return fn(*args, **kwargs) - - monkeypatch.setattr(tracing, "trace_call", fake_trace_call) - - @tracing.trace_function( - "answer-question", - metadata_factory=lambda value: {"value": value}, - ) - def decorated(value): - return value.upper() - - assert decorated("hello") == "HELLO" - assert captured == { - "name": "answer-question", - "run_type": "chain", - "metadata": {"value": "hello"}, - } - - -def test_trace_function_shields_metadata_factory_exceptions(monkeypatch, caplog): - captured = {} - - def fake_trace_call(name, fn, *args, run_type, metadata, **kwargs): - captured["metadata"] = metadata - return fn(*args, **kwargs) - - def failing_metadata_factory(value): - raise KeyError(value) - - monkeypatch.setattr(tracing, "trace_call", fake_trace_call) - - @tracing.trace_function( - "answer-question", - metadata_factory=failing_metadata_factory, - ) - def decorated(value): - return value.upper() - - with caplog.at_level(logging.WARNING, logger=tracing.__name__): - assert decorated("hello") == "HELLO" - - assert captured["metadata"] == {} - assert "Metadata factory failed for trace 'answer-question'" in caplog.text - assert "KeyError: 'hello'" in caplog.text diff --git a/backend/tests/test_vision_ocr_imports.py b/backend/tests/test_vision_ocr_imports.py deleted file mode 100644 index ce0cd7bb..00000000 --- a/backend/tests/test_vision_ocr_imports.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Tests for issue #591 — module-level OCR imports with a global ``HAS_OCR`` flag. - -These verify that PIL/pytesseract are imported once at module load rather than -inline on every ``_ocr_caption`` call, and that the hot path short-circuits on -the boolean flag instead of re-running an import/try-except on each image. -""" -import builtins -import importlib -import inspect - -from app.rag import vision - - -class _Boom: - """Any attribute access raises — proves the OCR backend is never touched.""" - - def __getattr__(self, _name): - raise AssertionError( - "OCR backend must not be accessed when HAS_OCR is False" - ) - - -def _fake_image(return_text): - """Build a stand-in ``PIL.Image`` whose pipeline yields ``return_text``.""" - - class _FakeImg: - def convert(self, mode): - assert mode == "RGB" - return self - - return type("FakeImage", (), {"open": staticmethod(lambda _buf: _FakeImg())}) - - -def _fake_pytesseract(return_text): - return type( - "FakePytesseract", - (), - {"image_to_string": staticmethod(lambda _img: return_text)}, - ) - - -# ── Module surface ─────────────────────────────────────────────────────────── - -def test_module_exposes_boolean_has_ocr_flag(): - assert hasattr(vision, "HAS_OCR") - assert isinstance(vision.HAS_OCR, bool) - - -def test_image_and_pytesseract_are_module_level_symbols(): - # Present as module globals regardless of availability (None when missing). - assert "Image" in vars(vision) - assert "pytesseract" in vars(vision) - - -def test_ocr_caption_has_no_inline_imports(): - """Regression guard: inline imports must not creep back into the hot path.""" - src = inspect.getsource(vision._ocr_caption) - assert "import pytesseract" not in src - assert "from PIL" not in src - assert "HAS_OCR" in src # the flag is what gates execution now - - -# ── Runtime behaviour ───────────────────────────────────────────────────────── - -def test_ocr_caption_short_circuits_when_unavailable(monkeypatch): - """HAS_OCR False → returns '' without ever touching Image/pytesseract.""" - monkeypatch.setattr(vision, "HAS_OCR", False) - monkeypatch.setattr(vision, "Image", _Boom(), raising=False) - monkeypatch.setattr(vision, "pytesseract", _Boom(), raising=False) - assert vision._ocr_caption(b"any-bytes") == "" - - -def test_ocr_caption_uses_module_objects_when_available(monkeypatch): - monkeypatch.setattr(vision, "HAS_OCR", True) - monkeypatch.setattr(vision, "Image", _fake_image(" hello world ")) - monkeypatch.setattr(vision, "pytesseract", _fake_pytesseract(" hello world ")) - assert vision._ocr_caption(b"img-bytes") == "hello world" - - -def test_ocr_caption_truncates_long_text(monkeypatch): - long_text = "x" * 600 - monkeypatch.setattr(vision, "HAS_OCR", True) - monkeypatch.setattr(vision, "Image", _fake_image(long_text)) - monkeypatch.setattr(vision, "pytesseract", _fake_pytesseract(long_text)) - result = vision._ocr_caption(b"img") - assert result.endswith("...") - assert len(result) == 503 # 500 chars + "..." - - -def test_ocr_caption_swallows_runtime_errors(monkeypatch): - """A pytesseract runtime error (e.g. missing tesseract binary) returns ''.""" - def _boom(_img): - raise RuntimeError("tesseract is not installed or not in PATH") - - monkeypatch.setattr(vision, "HAS_OCR", True) - monkeypatch.setattr(vision, "Image", _fake_image("")) - monkeypatch.setattr( - vision, - "pytesseract", - type("P", (), {"image_to_string": staticmethod(_boom)}), - ) - assert vision._ocr_caption(b"img") == "" - - -# ── Import-availability detection ───────────────────────────────────────────── - -def test_has_ocr_false_when_imports_missing(monkeypatch): - """Reloading with PIL/pytesseract blocked sets HAS_OCR=False and leaves the - backend symbols as None — and the hot path stays safe.""" - real_import = builtins.__import__ - - def _blocked_import(name, *args, **kwargs): - if name == "pytesseract" or name.split(".")[0] == "PIL": - raise ImportError(f"blocked for test: {name}") - return real_import(name, *args, **kwargs) - - monkeypatch.setattr(builtins, "__import__", _blocked_import) - reloaded = importlib.reload(vision) - try: - assert reloaded.HAS_OCR is False - assert reloaded.Image is None - assert reloaded.pytesseract is None - assert reloaded._ocr_caption(b"bytes") == "" - finally: - # Restore real import machinery and the genuine module state so the - # reloaded module object other tests share is left healthy. - monkeypatch.undo() - importlib.reload(reloaded) diff --git a/backend/tests/test_workspace_invite_email.py b/backend/tests/test_workspace_invite_email.py deleted file mode 100644 index aa4cceb2..00000000 --- a/backend/tests/test_workspace_invite_email.py +++ /dev/null @@ -1,120 +0,0 @@ -""" -Tests for send_workspace_invite_email — issue #442. -""" -from unittest.mock import call, patch - -import pytest - -from app.email_service import send_workspace_invite_email - - -WORKSPACE = "Research Team" -INVITE_LINK = "http://localhost:3000/invite?token=abc123" -EXPIRES = 72 -RECIPIENT = "newuser@example.com" - - -class TestSendWorkspaceInviteEmail: - def test_delegates_to_send_email(self): - """send_workspace_invite_email must call send_email exactly once.""" - with patch("app.email_service.send_email") as mock_send: - send_workspace_invite_email( - to=RECIPIENT, - workspace_name=WORKSPACE, - invite_link=INVITE_LINK, - expires_in_hours=EXPIRES, - ) - mock_send.assert_called_once() - - def test_subject_contains_workspace_name(self): - with patch("app.email_service.send_email") as mock_send: - send_workspace_invite_email( - to=RECIPIENT, - workspace_name=WORKSPACE, - invite_link=INVITE_LINK, - expires_in_hours=EXPIRES, - ) - _to, subject, _body = mock_send.call_args.args - assert WORKSPACE in subject - - def test_html_body_contains_invite_link(self): - with patch("app.email_service.send_email") as mock_send: - send_workspace_invite_email( - to=RECIPIENT, - workspace_name=WORKSPACE, - invite_link=INVITE_LINK, - expires_in_hours=EXPIRES, - ) - html = mock_send.call_args.kwargs.get("html") or "" - assert INVITE_LINK in html - - def test_html_body_contains_workspace_name(self): - with patch("app.email_service.send_email") as mock_send: - send_workspace_invite_email( - to=RECIPIENT, - workspace_name=WORKSPACE, - invite_link=INVITE_LINK, - expires_in_hours=EXPIRES, - ) - html = mock_send.call_args.kwargs.get("html") or "" - assert WORKSPACE in html - - def test_html_body_contains_expiry(self): - with patch("app.email_service.send_email") as mock_send: - send_workspace_invite_email( - to=RECIPIENT, - workspace_name=WORKSPACE, - invite_link=INVITE_LINK, - expires_in_hours=EXPIRES, - ) - html = mock_send.call_args.kwargs.get("html") or "" - assert str(EXPIRES) in html - - def test_personal_message_included_in_html(self): - message = "Looking forward to working with you!" - with patch("app.email_service.send_email") as mock_send: - send_workspace_invite_email( - to=RECIPIENT, - workspace_name=WORKSPACE, - invite_link=INVITE_LINK, - expires_in_hours=EXPIRES, - personal_message=message, - ) - html = mock_send.call_args.kwargs.get("html") or "" - assert message in html - - def test_no_personal_message_omits_block(self): - """Without a personal_message the optional block must not appear.""" - with patch("app.email_service.send_email") as mock_send: - send_workspace_invite_email( - to=RECIPIENT, - workspace_name=WORKSPACE, - invite_link=INVITE_LINK, - expires_in_hours=EXPIRES, - personal_message=None, - ) - html = mock_send.call_args.kwargs.get("html") or "" - # The placeholder block rendered for None should be empty / absent - assert "border-left:4px solid #4f46e5" not in html - - def test_plain_text_body_contains_invite_link(self): - with patch("app.email_service.send_email") as mock_send: - send_workspace_invite_email( - to=RECIPIENT, - workspace_name=WORKSPACE, - invite_link=INVITE_LINK, - expires_in_hours=EXPIRES, - ) - _to, _subject, plain_body = mock_send.call_args.args - assert INVITE_LINK in plain_body - - def test_recipient_passed_to_send_email(self): - with patch("app.email_service.send_email") as mock_send: - send_workspace_invite_email( - to=RECIPIENT, - workspace_name=WORKSPACE, - invite_link=INVITE_LINK, - expires_in_hours=EXPIRES, - ) - to_arg = mock_send.call_args.args[0] - assert to_arg == RECIPIENT diff --git a/backend/tests/test_workspaces.py b/backend/tests/test_workspaces.py index 6a5085f0..8bf5610a 100644 --- a/backend/tests/test_workspaces.py +++ b/backend/tests/test_workspaces.py @@ -11,7 +11,7 @@ def test_workspace_invite_requires_admin(client, db_session, user): ) assert response.status_code == 403 - assert response.json()["error"]["message"] == "Admin access required" + assert response.json()["detail"] == "Admin access required" def test_workspace_invite_creates_invitation_and_sends_email(client, db_session, monkeypatch): @@ -27,12 +27,12 @@ def test_workspace_invite_creates_invitation_and_sends_email(client, db_session, sent = {} - def fake_send_workspace_invite_email(to, workspace_name, invite_link, expires_in_hours, personal_message=None): + def fake_send_email(to, subject, body, html=None): sent["to"] = to - sent["workspace_name"] = workspace_name - sent["invite_link"] = invite_link + sent["subject"] = subject + sent["body"] = body - monkeypatch.setattr("app.routes.workspaces.send_workspace_invite_email", fake_send_workspace_invite_email) + monkeypatch.setattr("app.routes.workspaces.send_email", fake_send_email) token = create_access_token(admin.id) response = client.post( @@ -49,8 +49,7 @@ def fake_send_workspace_invite_email(to, workspace_name, invite_link, expires_in assert payload["invite_link"].startswith("http") assert "token=" in payload["invite_link"] assert sent["to"] == "invitee@example.com" - assert sent["workspace_name"] == payload["workspace_name"] - assert "token=" in sent["invite_link"] + assert "Invitation to join workspace" in sent["subject"] invitation = db_session.query(WorkspaceInvitation).filter_by(email="invitee@example.com").first() assert invitation is not None diff --git a/config.py b/config.py new file mode 100644 index 00000000..1e1d059e --- /dev/null +++ b/config.py @@ -0,0 +1,25 @@ +import os +from dotenv import load_dotenv + +load_dotenv() + +# ── App Config ─────────────────────────────────────── +SECRET_KEY = os.getenv("SECRET_KEY", "your_secret_key_here") +ENCRYPTION_KEY = os.getenv("ENCRYPTION_KEY", b"T4tQj_3jK7z_gBqxZ1j_aGj8sFpXv_f4jZ8Rj9sPqG0=") +MONGO_URI = os.getenv("MONGO_URI", "mongodb://localhost:27017/rag_app") + +# ── Upload Config ──────────────────────────────────── +UPLOAD_FOLDER = "uploads" +ALLOWED_EXTENSIONS = {"pdf", "docx", "txt", "md"} + +# ── RAG Config ─────────────────────────────────────── +TOP_K = 5 +CHUNK_SIZE = 500 +CHUNK_OVERLAP = 50 + +# ── Groq Config ────────────────────────────────────── +GROQ_MODEL = "llama-3.3-70b-versatile" + +# ── Google OAuth Config ────────────────────────────── +GOOGLE_CLIENT_ID = os.getenv("GOOGLE_CLIENT_ID") +GOOGLE_CLIENT_SECRET = os.getenv("GOOGLE_CLIENT_SECRET") \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 4638aa8c..96ee7b7d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,18 +1,11 @@ version: '3.8' -x-logging: &default-logging - driver: "json-file" - options: - max-size: "10m" - max-file: "5" - services: # Redis broker/result backend for Celery document processing redis: image: redis:7-alpine container_name: pdf_rag_redis restart: unless-stopped - logging: *default-logging ports: - "6379:6379" healthcheck: @@ -27,7 +20,6 @@ services: image: postgres:16-alpine container_name: pdf_rag_postgres restart: unless-stopped - logging: *default-logging environment: POSTGRES_DB: ${POSTGRES_DB:-pdf_rag} POSTGRES_USER: ${POSTGRES_USER:-pdf_rag_user} @@ -44,45 +36,10 @@ services: retries: 5 start_period: 10s - # ── Application (CPU profile) ──────────────────────────── + # ── Application ────────────────────────────────────────── app: build: . container_name: pdf_rag_app - profiles: ["cpu"] - logging: *default-logging - ports: - - "7860:7860" - volumes: - - app_data:/app/data - environment: - - SECRET_KEY=${SECRET_KEY:-dev-secret-key-change-me} - - HF_TOKEN=${HF_TOKEN} - - DATABASE_URL=postgresql+psycopg://${POSTGRES_USER:-pdf_rag_user}:${POSTGRES_PASSWORD:-pdf_rag_pass}@postgres:5432/${POSTGRES_DB:-pdf_rag} - - UPLOAD_DIR=/app/data/uploads - - CHROMA_PERSIST_DIR=/app/data/chroma_db - - GRAPH_PERSIST_DIR=/app/data/graphs - - CELERY_BROKER_URL=redis://redis:6379/0 - - CELERY_RESULT_BACKEND=redis://redis:6379/1 - - DEVICE=cpu - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - restart: unless-stopped - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:7860/api/health"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 60s - - # ── Application (GPU profile) ──────────────────────────── - app-gpu: - build: . - container_name: pdf_rag_app - profiles: ["gpu"] - logging: *default-logging ports: - "7860:7860" volumes: @@ -90,20 +47,12 @@ services: environment: - SECRET_KEY=${SECRET_KEY:-dev-secret-key-change-me} - HF_TOKEN=${HF_TOKEN} - - DATABASE_URL=postgresql+psycopg://${POSTGRES_USER:-pdf_rag_user}:${POSTGRES_PASSWORD:-pdf_rag_pass}@postgres:5432/${POSTGRES_DB:-pdf_rag} + - DATABASE_URL=postgresql://${POSTGRES_USER:-pdf_rag_user}:${POSTGRES_PASSWORD:-pdf_rag_pass}@postgres:5432/${POSTGRES_DB:-pdf_rag} - UPLOAD_DIR=/app/data/uploads - CHROMA_PERSIST_DIR=/app/data/chroma_db - GRAPH_PERSIST_DIR=/app/data/graphs - CELERY_BROKER_URL=redis://redis:6379/0 - CELERY_RESULT_BACKEND=redis://redis:6379/1 - - DEVICE=cuda - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: all - capabilities: [gpu] depends_on: postgres: condition: service_healthy @@ -117,25 +66,24 @@ services: retries: 3 start_period: 60s - # Celery worker (CPU profile) + # Celery worker for document extraction, chunking, embeddings, and vector storage worker: build: . container_name: pdf_rag_worker - profiles: ["cpu"] - logging: *default-logging - command: celery -A app.celery_app.celery_app worker --loglevel=info + command: > + sh -c "cd /app/backend && + celery -A app.celery_app.celery_app worker --loglevel=info" volumes: - app_data:/app/data environment: - SECRET_KEY=${SECRET_KEY:-dev-secret-key-change-me} - HF_TOKEN=${HF_TOKEN} - - DATABASE_URL=postgresql+psycopg://${POSTGRES_USER:-pdf_rag_user}:${POSTGRES_PASSWORD:-pdf_rag_pass}@postgres:5432/${POSTGRES_DB:-pdf_rag} + - DATABASE_URL=postgresql://${POSTGRES_USER:-pdf_rag_user}:${POSTGRES_PASSWORD:-pdf_rag_pass}@postgres:5432/${POSTGRES_DB:-pdf_rag} - UPLOAD_DIR=/app/data/uploads - CHROMA_PERSIST_DIR=/app/data/chroma_db - GRAPH_PERSIST_DIR=/app/data/graphs - CELERY_BROKER_URL=redis://redis:6379/0 - CELERY_RESULT_BACKEND=redis://redis:6379/1 - - DEVICE=cpu depends_on: postgres: condition: service_healthy @@ -143,66 +91,12 @@ services: condition: service_healthy restart: unless-stopped - # Celery worker (GPU profile) - worker-gpu: - build: . - container_name: pdf_rag_worker - profiles: ["gpu"] - logging: *default-logging - command: celery -A app.celery_app.celery_app worker --loglevel=info - volumes: - - app_data:/app/data - environment: - - SECRET_KEY=${SECRET_KEY:-dev-secret-key-change-me} - - HF_TOKEN=${HF_TOKEN} - - DATABASE_URL=postgresql+psycopg://${POSTGRES_USER:-pdf_rag_user}:${POSTGRES_PASSWORD:-pdf_rag_pass}@postgres:5432/${POSTGRES_DB:-pdf_rag} - - UPLOAD_DIR=/app/data/uploads - - CHROMA_PERSIST_DIR=/app/data/chroma_db - - GRAPH_PERSIST_DIR=/app/data/graphs - - CELERY_BROKER_URL=redis://redis:6379/0 - - CELERY_RESULT_BACKEND=redis://redis:6379/1 - - DEVICE=cuda - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: all - capabilities: [gpu] - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - restart: unless-stopped - - # ── Frontend (Next.js static export served by nginx) ───── - frontend: - build: - context: ./frontend - dockerfile: Dockerfile - args: - NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:-http://localhost:7860} - container_name: pdf_rag_frontend - profiles: ["cpu", "gpu"] - logging: *default-logging - ports: - - "3000:3000" - restart: unless-stopped - healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:3000"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 20s - # ── pgAdmin (optional — for local DB inspection) ───────── pgadmin: image: dpage/pgadmin4:latest container_name: pdf_rag_pgadmin restart: unless-stopped - profiles: ["debug"] - logging: *default-logging + profiles: ["debug"] # only starts when: docker compose --profile debug up environment: PGADMIN_DEFAULT_EMAIL: admin@local.dev PGADMIN_DEFAULT_PASSWORD: admin diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index c512492a..f2351553 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -4,39 +4,7 @@ This guide gives contributors a map of the PDF-Assistant-RAG runtime before they change an endpoint, storage model, or RAG step. The README keeps the product overview; this page focuses on how requests move through the system. ---- - -## Table of Contents - -1. [System Overview](#system-overview) -2. [Backend Architecture](#backend-architecture) - - [Route Structure](#route-structure) - - [Business Logic Layer](#business-logic-layer) - - [RAG Pipeline](#rag-pipeline) - - [Authentication Flow](#authentication-flow) - - [Data Models & Relationships](#data-models--relationships) - - [Background Tasks](#background-tasks) -3. [Frontend Architecture](#frontend-architecture) - - [Pages & Routing (Next.js App Router)](#pages--routing-nextjs-app-router) - - [State Management (Zustand)](#state-management-zustand) - - [API Client Layer](#api-client-layer) - - [Component Tree](#component-tree) - - [Data Flow: Action → Store → API → Response → UI](#data-flow-action--store--api--response--ui) -4. [Infrastructure](#infrastructure) - - [Docker Multi-Stage Build](#docker-multi-stage-build) - - [CI/CD Pipelines](#cicd-pipelines) - - [Environment Configuration](#environment-configuration) -5. [Data Flow Diagrams](#data-flow-diagrams) - - [Upload → Process → Query](#upload--process--query) - - [Login → Token Refresh → OAuth](#login--token-refresh--oauth) - - [Question → RAG → Streamed Answer](#question--rag--streamed-answer) -6. [Data Ownership & Boundaries](#data-ownership--boundaries) -7. [Swagger & OpenAPI Notes](#swagger--openapi-notes) -8. [Local Contributor Checklist](#local-contributor-checklist) - ---- - -## System Overview +## Runtime Topology ```mermaid flowchart LR @@ -48,8 +16,6 @@ flowchart LR RAG["RAG services
chunking, embeddings, reranking"] LLM["HuggingFace inference
answer generation"] GitHub["GitHub API
public repo stats"] - Redis["Redis broker
Celery task queue"] - Worker["Celery worker
async document processing"] Browser -->|"JWT + REST"| API Browser -->|"SSE chat stream"| API @@ -57,14 +23,9 @@ flowchart LR API --> Uploads API --> Chroma API --> RAG - API --> GitHub - API --> Redis - Redis --> Worker - Worker --> Uploads - Worker --> Chroma - Worker --> SQL RAG --> Chroma RAG --> LLM + API --> GitHub ``` The frontend is a Next.js application that talks to the FastAPI backend. In @@ -73,730 +34,18 @@ development it usually runs on `http://localhost:3000`; the backend runs on In production the backend can also serve the exported frontend from `frontend/out` when that directory exists. -Redis acts as both the Celery broker (task queue) and result backend. The -Celery worker handles expensive document processing (text extraction, -chunking, embedding, graph building) asynchronously so the API stays -responsive. - ---- - -## Backend Architecture - -### Route Structure - -All API routes are mounted under `/api/v1` in `backend/app/main.py`: - -```python -app.include_router(auth_router, prefix="/api/v1") -app.include_router(documents_router, prefix="/api/v1") -app.include_router(chat_router, prefix="/api/v1") -app.include_router(github_router, prefix="/api/v1") -app.include_router(admin_router, prefix="/api/v1") -app.include_router(workspaces_router, prefix="/api/v1") -``` - -| Route group | Prefix | Main file | Responsibility | -| --- | --- | --- | --- | -| Auth | `/api/v1/auth` | `routes/auth.py` | Registration, login, Google OAuth, JWT refresh/verify, email verification, API key management, profile update, password change | -| Documents | `/api/v1/documents` | `routes/documents.py` | Upload (multipart + URL), list, status polling, serve PDF, rename, update metadata, soft-delete, chunk settings, table extraction | -| Chat | `/api/v1/chat` | `routes/chat.py` | Ask (non-streaming), ask/stream (SSE), session CRUD, history, message feedback, share message | -| GitHub | `/api/v1/github/stats` | `routes/github.py` | Cached public repo stats for landing page | -| Admin | `/api/v1/admin` | `routes/admin.py` | User inventory, operational stats, system metrics | -| Workspaces | `/api/v1/workspaces` | `routes/workspaces.py` | Workspace invitations, collaborative spaces | -| Profile | `/api/v1/profile` | `routes/profile.py` | User profile display name & avatar updates | -| Health | `/health`, `/api/health` | `main.py` | Lightweight health check (API, SQL, Chroma) | - -**FastAPI route files follow a consistent pattern:** -1. Route handler receives request + dependencies (DB session, current user) -2. Input validation via Pydantic schemas -3. Business logic inline or delegated to `services/` -4. Response serialization via `response_model` - -### Business Logic Layer - -The project does **not** have a formal service layer for all operations. -Business logic lives in two places: - -1. **Inline in route handlers** — most CRUD operations (auth, chat sessions, - admin) are handled directly in route files for simplicity. -2. **`app/services/` directory** — complex or shared logic is extracted: - - `document_ingestion.py` — handles file parsing, table extraction, and - orchestrates the full ingestion pipeline - - `layout_parser.py` — advanced PDF layout analysis (headings, tables, - figures) using a hierarchy of parser classes - -``` -backend/app/services/ -├── __init__.py -├── document_ingestion.py # Full ingestion pipeline orchestration -├── layout_parser.py # Advanced PDF layout analysers -└── drive_sync.py # Google Drive background sync -``` - -The RAG pipeline lives entirely in `app/rag/`: - -``` -backend/app/rag/ -├── __init__.py -├── agent.py # LangGraph agent orchestrating retrieval + generation -├── bm25.py # BM25 keyword retrieval (complements vector search) -├── chunker.py # Text chunking strategies (recursive, semantic) -├── embeddings.py # HuggingFace embedding model (all-MiniLM-L6-v2) -├── graph_builder.py # Knowledge graph extraction from document chunks -├── graph_retriever.py# GraphRAG traversal for relationship-aware retrieval -├── prompts.py # LLM prompt templates -├── retriever.py # Two-stage hybrid retrieval + cross-encoder reranking -├── security.py # Prompt injection detection -├── summarizer.py # Document summarization from ingested chunks -├── tools.py # LangGraph agent tool definitions -├── tracing.py # LangSmith trace helpers -├── vectorstore.py # ChromaDB client, CRUD for vector chunks -└── vision.py # Image captioning for scanned PDF figures -``` - -### RAG Pipeline - -```mermaid -flowchart TD - A["User uploads PDF/DOCX/TXT/MD"] - B["Validate file (extension, MIME, size, parser check)"] - C["Persist to upload directory"] - D["Create Document row (status: pending)"] - E["Queue Celery ingestion task"] - F["Celery Worker: extract text & tables"] - G["Celery Worker: chunk text (recursive + semantic)"] - H["Celery Worker: build knowledge graph entities & relationships"] - I["Celery Worker: generate summary"] - J["Celery Worker: embed chunks with all-MiniLM-L6-v2"] - K["Store chunks in ChromaDB with user/doc metadata"] - L["Update Document row (status: ready, page/chunk count, summary)"] - - A --> B - B --> C - C --> D - D --> E - E --> F - F --> G - G --> H - H --> I - I --> J - J --> K - K --> L -``` - -**At query time (chat):** - -```mermaid -flowchart TD - Q["User asks a question"] - V["Embed query with all-MiniLM-L6-v2"] - S["Hybrid retrieval: ChromaDB (vector) + BM25 (keyword)"] - E["Ensemble: combine & deduplicate candidates"] - R["Rerank with cross-encoder (ms-marco-MiniLM-L6-v2)"] - G["Optional: GraphRAG for relationship-aware context"] - P["Build prompt with selected context + conversation history"] - L["Query HuggingFace Inference API (Qwen2.5-72B)"] - A["Return answer + source citations"] - - Q --> V - V --> S - S --> E - E --> R - R --> P - G --> P - P --> L - L --> A -``` - -The retriever uses a **two-stage strategy**: -1. **Stage 1 — Hybrid Ensemble**: Combines ChromaDB vector similarity search - (dense) with BM25 keyword retrieval (sparse). Configurable `TOP_K_RETRIEVAL` - (default: 10 candidates). -2. **Stage 2 — Cross-Encoder Reranking**: Re-scores candidates with - `cross-encoder/ms-marco-MiniLM-L-6-v2` and keeps the top - `TOP_K_RERANK` (default: 5). If the reranker model fails to load, - the pipeline falls back to embedding-only retrieval. - -Embeddings use `sentence-transformers/all-MiniLM-L6-v2` (384 dimensions), -loaded once at startup and shared across all users. - -### Authentication Flow - -```mermaid -sequenceDiagram - participant User as User/Browser - participant API as FastAPI /api/v1/auth - participant DB as SQL Database - participant HF as HuggingFace (optional) - - %% Password Registration - User->>API: POST /register { username, email, password } - API->>API: Hash password (bcrypt) - API->>DB: Create User row - API->>API: Generate verification token - alt SMTP configured - API->>User: Send verification email - else dev mode - API-->>User: Return verification_url in response - end - API-->>User: { message, email, verification_url? } - - %% Email Verification - User->>API: GET /verify-email?token=xxx - API->>DB: Hash token, find & verify User - API-->>User: Redirect to dashboard - - %% Login - User->>API: POST /login { email, password } - API->>API: Verify password (bcrypt.checkpw) - API->>API: Create access_token (15 min) + refresh_token (7 days) - API-->>User: { access_token, refresh_token, user } - - %% API Key Auth - User->>API: GET /documents (with header: Authorization: Bearer pdf_rag_xxx) - API->>API: Detect pdf_rag_ prefix, hash key with SHA256 - API->>DB: Look up ApiKey by hashed_key - API-->>User: Response (if key is active) - - %% Google OAuth - User->>API: POST /google { id_token } - API->>API: Verify id_token with Google (httpx) - alt New user - API->>DB: Create User row - end - API->>API: Create JWT tokens - API-->>User: { access_token, refresh_token, user } - - %% Token Refresh - User->>API: POST /refresh { refresh_token } - API->>API: Decode + validate refresh token - API->>API: Issue new access_token (and optional new refresh_token) - API-->>User: { access_token, refresh_token? } - - %% HuggingFace Token - User->>API: PUT /hf-token { hf_token } - API->>DB: Encrypt & store in User.hf_token (Fernet AES) - API-->>User: { user with hf_token updated } -``` - -**Key authentication mechanisms:** - -| Method | Mechanism | Token Format | Expiry | -|--------|-----------|-------------|--------| -| Password | bcrypt hashing + JWT | Bearer `access_token` | 15 min | -| Refresh | JWT with `type: "refresh"` | Rotated on use | 7 days | -| API Key | SHA256 hash lookup | `pdf_rag_...` prefix | Manual revoke | -| Google OAuth | ID token verification via Google API | Auto-creates JWT | Per-session | - -The `get_current_user` FastAPI dependency (in `app/auth.py`) handles all auth -methods transparently: -1. Checks `Authorization: Bearer` header (JWT or API key) -2. Falls back to secure cookie (`access_token` cookie) -3. API keys are detected by the `pdf_rag_` prefix and validated via SHA256 hash -4. Returns `403 Forbidden` for admin-only routes via `get_admin_user` - -**Email verification flow:** -- On registration, a 32-byte random token is generated, SHA256-hashed, and - stored in `verification_token_hash`. -- If SMTP is configured, a verification email is sent with the link. -- In development without SMTP, the response includes a `verification_url`. -- Tokens expire after `EMAIL_VERIFICATION_TOKEN_EXPIRE_HOURS` (default: 24). - -### Data Models & Relationships - -``` -┌──────────────────────────┐ -│ User │ -├──────────────────────────┤ -│ id (UUID, PK) │ -│ username, email │ -│ hashed_password │ -│ role (user | admin) │ -│ is_verified │ -│ hf_token (encrypted) │ -│ display_name, avatar_url │ -│ created_at, last_login │ -├──────────────────────────┤ -│ 1 ──< Document │ -│ 1 ──< ChatSession │ -│ 1 ──< ChatMessage │ -│ 1 ──< ApiKey │ -│ 1 ──< DriveConnection │ -└──────────────────────────┘ - -┌──────────────────────────┐ -│ Document │ -├──────────────────────────┤ -│ id (UUID, PK) │ -│ user_id (FK → User) │ -│ filename, original_name │ -│ file_size, page_count │ -│ chunk_count │ -│ status (pending|processing|ready|failed) │ -│ summary │ -│ uploaded_at, last_accessed_at │ -│ is_deleted, deleted_at │ -│ drive_file_id, drive_folder_id │ -├──────────────────────────┤ -│ * ──1 User │ -│ 1 ──< ChatMessage │ -└──────────────────────────┘ - -┌──────────────────────────┐ -│ ChatSession │ -├──────────────────────────┤ -│ id (UUID, PK) │ -│ user_id (FK → User) │ -│ title │ -│ created_at │ -├──────────────────────────┤ -│ * ──1 User │ -│ 1 ──< ChatMessage │ -└──────────────────────────┘ - -┌──────────────────────────┐ -│ ChatMessage │ -├──────────────────────────┤ -│ id (UUID, PK) │ -│ user_id (FK → User) │ -│ document_id (FK → Doc, nullable) │ -│ session_id (FK → Session, nullable) │ -│ role (user | assistant) │ -│ content │ -│ sources_json (JSON text) │ -│ feedback (up | down | null) │ -│ created_at │ -├──────────────────────────┤ -│ * ──1 User │ -│ * ──1 Document │ -│ * ──1 ChatSession │ -│ 1 ──0..1 SharedMessage │ -└──────────────────────────┘ - -┌──────────────────────────┐ -│ ApiKey │ -├──────────────────────────┤ -│ id (UUID, PK) │ -│ user_id (FK → User) │ -│ key_prefix │ -│ hashed_key (SHA256) │ -│ is_active │ -│ created_at, last_used_at │ -├──────────────────────────┤ -│ * ──1 User │ -└──────────────────────────┘ - -┌──────────────────────────┐ -│ SharedMessage │ -├──────────────────────────┤ -│ id (UUID, PK) │ -│ message_id (FK → ChatMessage, unique) │ -│ created_at │ -├──────────────────────────┤ -│ * ──1 ChatMessage │ -└──────────────────────────┘ - -┌──────────────────────────┐ -│ DriveConnection │ -├──────────────────────────┤ -│ id (UUID, PK) │ -│ user_id (FK → User) │ -│ folder_id │ -│ credentials_json │ -│ enabled │ -│ last_synced_at │ -└──────────────────────────┘ - -┌──────────────────────────┐ -│ WorkspaceInvitation │ -├──────────────────────────┤ -│ id (UUID, PK) │ -│ email │ -│ token_hash (SHA256) │ -│ inviter_id (FK → User) │ -│ workspace_name │ -│ expires_at, accepted_at │ -└──────────────────────────┘ -``` - -**Key design decisions:** -- UUIDs are stored as strings for SQLite compatibility, native UUID type on - PostgreSQL via the `GUID` type decorator. -- `hf_token` is encrypted at rest using Fernet (AES via `cryptography`), - derived from `SECRET_KEY`. -- Documents use **soft-delete** (`is_deleted` flag) to preserve references. -- Chunk vectors in ChromaDB are keyed by `document_id` and `user_id` for - multi-tenant isolation. -- `sources_json` stores source citations as a JSON string (not a relational - table) for simplicity — sources are always read/written as a unit. - -### Background Tasks - -The application uses two background processing mechanisms: - -**1. Celery Workers (async document ingestion)** - -```mermaid -flowchart LR - API["API: POST /upload"] - Redis["Redis broker"] - Worker["Celery worker"] - - API -->|"Queue task"| Redis - Redis -->|"Deliver"| Worker - Worker -->|"Extract, chunk, embed"| Chroma - Worker -->|"Save metadata"| SQL -``` - -- **Broker/Backend:** Redis (`CELERY_BROKER_URL`, `CELERY_RESULT_BACKEND`) -- **Task definition:** `app/tasks.py` — `process_document()` function -- **Worker command:** `celery -A app.celery_app worker --loglevel=info` -- Processing status is tracked in `Document.status` (pending → processing → - ready/failed) - -**2. In-process background loops (lightweight maintenance)** - -```python -# In main.py lifespan — runs asyncio.create_task -async def document_cleanup_job(): - """Periodically purge documents not accessed in 30 days.""" - while True: - # Query expired documents, delete files + vectors + DB rows - await asyncio.sleep(86400) # Every 24 hours -``` - -**3. APScheduler (periodic sync jobs)** - -Configured in `app/scheduler.py` via `start_scheduler()`: -- Google Drive sync (`DRIVE_SYNC_ENABLED` + `DRIVE_SYNC_INTERVAL_MINUTES`) -- Metrics export (Prometheus endpoint at `/metrics`) - ---- - -## Frontend Architecture - -### Pages & Routing (Next.js App Router) - -``` -frontend/src/app/ -├── layout.tsx # Root layout: ThemeProvider, AuthProvider, i18n, Tooltip -├── page.tsx # Landing page (hero, features, GitHub stats, footer) -├── globals.css # Tailwind v4 global styles + theme definitions -├── login/ -│ └── page.tsx # Login page (email/password + Google OAuth) -├── register/ -│ └── page.tsx # Registration page -├── verify-email/ -│ └── page.tsx # Email verification handler -├── dashboard/ -│ └── page.tsx # Main dashboard (chat interface + document sidebar) -├── drive/ -│ └── page.tsx # Google Drive integration page -├── admin/ -│ └── page.tsx # Admin panel (users, system stats) -├── share/ -│ └── [id]/ -│ └── page.tsx # Public shared message view -├── privacy/ -│ └── page.tsx # Privacy policy (static, prose layout) -└── terms/ - └── page.tsx # Terms of service (static, prose layout) -``` - -**Layout hierarchy:** - -``` - (RootLayout) - └── (next-themes — light/dark/ocean/forest/sunset) - └── (JWT token sync, auth events) - └── (react-i18next) - └── (@base-ui/react tooltip context) - └── (sonner toast notifications) -``` - -### State Management (Zustand) - -Two Zustand stores manage client-side state: - -**1. `auth-store.ts` — Authentication state** -```typescript -interface AuthStore { - user: AuthUser | null; // Current user profile - token: string | null; // JWT access token - loading: boolean; // Initial auth check in progress - initialized: boolean; // Auth initialization complete - - // Actions - login(email, password) // POST /api/v1/auth/login - loginWithGoogle(idToken) // POST /api/v1/auth/google - register(username, email, password) // POST /api/v1/auth/register - logout() // POST /api/v1/auth/logout + clear tokens - initializeAuth() // GET /api/v1/auth/me (restore session) - setHfToken(hfToken) // PUT /api/v1/auth/hf-token - syncTokensRefreshed(detail) // Handle auth:tokens-refreshed event - syncLoggedOut() // Handle auth:logged-out event -} -``` - -**2. `chat-store.ts` — Chat state** -```typescript -interface ChatStore { - messages: ChatMsg[]; // Current session messages - input: string; // Chat input text - streaming: boolean; // SSE stream in progress - isTyping: boolean; // Typing indicator (API generating) - historyLoading: boolean; // Loading session history - sessions: ChatSession[]; // All user sessions - activeSessionId: string | null; // Currently active session - - // Actions - fetchSessions() // GET /api/v1/chat/sessions - createSession(title) // POST /api/v1/chat/sessions - renameSession(id, title) // PUT /api/v1/chat/sessions/{id} - deleteSession(id) // DELETE /api/v1/chat/sessions/{id} - fetchSessionHistory(id) // GET /api/v1/chat/history/session/{id} - resetChat() // Reset all state -} -``` - -**Store pattern:** Each store uses Zustand's `create()` with the setter/getter -pattern. A generic `resolveValue` helper supports both direct values and -updater functions for `setMessages`, `setInput`, etc. - -### API Client Layer +## Backend Route Groups -`src/lib/api.ts` — A thin wrapper around `fetch()` that provides: +| Route group | Prefix | Responsibility | +| --- | --- | --- | +| Auth | `/api/v1/auth` | Registration, login, Google sign-in, JWT refresh, and profile state. | +| Documents | `/api/v1/documents` | File validation, upload records, background ingestion, status polling, file serving, deletion, and metadata updates. | +| Chat | `/api/v1/chat` | RAG questions, SSE streaming, chat sessions, history, exports, and shared answer links. | +| Admin | `/api/v1/admin` | Admin-only operational stats and user inventory. | +| GitHub | `/api/v1/github/stats` | Cached public repository statistics for the landing page. | +| Health | `/health`, `/api/health` | Lightweight service health checks for API, SQL, and Chroma availability. | -```typescript -class ApiClient { - // Typed HTTP methods with auto-refresh - async get(path, options?) // GET request - async post(path, body?, options?) // POST request - async put(path, body?, options?) // PUT request - async patch(path, body?, options?) // PATCH request - async delete(path, options?) // DELETE request - async postForm(path, formData, options?) // Multipart form upload - - // SSE streaming - async *streamPost(path, body) // POST → SSE stream (AsyncGenerator) - - // Utilities - getPdfUrl(documentId) // Construct PDF download URL with token -} -``` - -**Key features:** -- Automatic JWT token injection from `localStorage` -- Transparent 401 → token refresh → retry (prevents race conditions with a - mutex guard on `refreshPromise`) -- Structured error messages from backend `{ detail }` payloads -- Connection error detection (TypeError → user-friendly message) -- Dispatches `auth:tokens-refreshed` and `auth:logged-out` custom events for - store synchronization - -### Component Tree - -``` -frontend/src/components/ -├── auth/ -│ ├── AuthProvider.tsx # Auth context: listens to token events -│ ├── HfTokenModal.tsx # HuggingFace token configuration modal -│ └── ApiKeyManager.tsx # API key management dialog -│ -├── chat/ -│ ├── ChatPanel.tsx # Main chat container -│ ├── MessageBubble.tsx # Single message (markdown, copy, share, speech, feedback) -│ ├── SourceCard.tsx # Source citations card (collapsible, confidence badges) -│ └── WelcomeScreen.tsx # Landing placeholder when no messages -│ -├── document/ -│ ├── DocumentSidebar.tsx # Document list sidebar with upload -│ ├── FileUploader.tsx # Drag-and-drop file upload zone -│ └── DocumentTable.tsx # Document table with status icons -│ -├── layout/ -│ ├── ThemeProvider.tsx # next-themes wrapper with custom themes -│ ├── Sidebar.tsx # Navigation sidebar -│ ├── Navbar.tsx # Top navigation bar -│ └── Footer.tsx # Landing page footer -│ -├── providers/ -│ └── I18nProvider.tsx # react-i18next initialization -│ -├── ui/ # Base UI primitives (shadcn-style wrappers) -│ ├── button.tsx # Button (@base-ui/react/button + CVA) -│ ├── badge.tsx # Badge component -│ ├── tooltip.tsx # Tooltip (@base-ui/react/tooltip) -│ ├── dialog.tsx # Dialog (@base-ui/react/dialog) -│ ├── input.tsx # Input with base-ui -│ ├── dropdown-menu.tsx # Dropdown menu -│ ├── confirm-dialog.tsx # Confirmation dialog (danger/warning/default variants) -│ └── ... # Other primitives -│ -├── DriveFolderSelector.tsx # Google Drive folder picker -└── EmptyState.tsx # Generic empty state display -``` - -**UI component design:** -- All UI primitives are wrappers around `@base-ui/react` (v1.4.1) -- Variants managed via `class-variance-authority` (CVA) -- Class merging via `tailwind-merge` + `clsx` -- Icons from `lucide-react` -- Styling with Tailwind CSS v4 + `tw-animate-css` for animations - -### Data Flow: Action → Store → API → Response → UI - -```mermaid -sequenceDiagram - participant UI as React Component - participant Store as Zustand Store - participant API as ApiClient - participant Backend as FastAPI Backend - - %% Read Flow - UI->>Store: Call store action (e.g., fetchSessions) - Store->>API: api.get("/api/v1/chat/sessions") - API->>API: Inject JWT from localStorage - API->>Backend: fetch() with Bearer token - Backend-->>API: JSON response - API-->>Store: Typed response (ChatSession[]) - Store->>Store: Update state (set({ sessions: data })) - Store-->>UI: React re-render via Zustand subscription - - %% Write Flow - UI->>Store: Call mutating action (e.g., renameSession) - Store->>API: api.put("/api/v1/chat/sessions/{id}", { title }) - API->>Backend: fetch() PUT with JSON body - Backend-->>API: Updated session JSON - Store->>Store: Map over sessions, replace matching entry - Store-->>UI: React re-render - - %% Streaming Flow - UI->>API: api.streamPost("/api/v1/chat/ask/stream", { question }) - API->>Backend: fetch() POST → SSE stream - loop For each SSE event - Backend-->>API: data: { type: "token", data: "Hello" } - API-->>UI: yield { type: "token", data: "Hello" } - UI->>UI: Append token to message content - end - Backend-->>API: data: { type: "done", data: { sources: [...] } } - API-->>UI: yield final event with sources - UI->>Store: Save complete message with sources -``` - ---- - -## Infrastructure - -### Docker Multi-Stage Build - -The `Dockerfile` uses three stages to minimise the final image size: - -```mermaid -flowchart LR - A["Stage 1: frontend-builder
node:20-alpine
npm ci → npm run build"] - B["Stage 2: python-builder
python:3.11-slim
pip install -r requirements.txt"] - C["Stage 3: runtime
python:3.11-slim
app code + frontend build + venv"] - - A -->|"frontend/out"| C - B -->|"/opt/venv"| C -``` - -1. **frontend-builder** — Builds Next.js static export (`frontend/out`) -2. **python-builder** — Installs Python deps in a virtualenv, including spaCy - model `en_core_web_sm` and system packages (`libmagic1`, `build-essential`) -3. **runtime** — Copies only the venv and app code. Runs as user 1000 - (HuggingFace Spaces requirement). Exposes port 7860. - -```bash -CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"] -``` - -The `docker-compose.yml` provides a full local stack with Redis and Celery worker. - -### CI/CD Pipelines - -The project uses GitHub Actions with these workflows (all in `.github/workflows/`): - -| Workflow | File | Trigger | What it does | -|----------|------|---------|-------------| -| **CI — Dev Branch** | `ci.yml` | Push/PR to `dev` | Backend lint (flake8), import check, pytest (40% coverage), CodeQL analysis, frontend type-check (tsc), ESLint, Vitest, Next.js build, PR size gate | -| **E2E Tests** | `e2e.yml` | PR to `dev` | Playwright E2E tests against full stack | -| **Sync Issue Labels** | `sync-issue-labels.yml` | `opened` → PR | Copies labels from referenced issue to PR | -| **GSSOC Welcome** | `gssoc-welcome.yml` | Issue/PR open | Welcome message for GSSoC contributors | -| **Deploy** | `deploy.yml` | Push to `main` | HuggingFace Spaces deployment | -| **DevSecOps** | `devsecops.yml` | Push/PR to `dev` | Additional security scanning | - -**CI checks that must pass before merge:** -1. 🐍 Backend lint & import check (flake8 errors only) -2. 🔎 CodeQL security analysis (fails on severity ≥ 9.0) -3. ⚛️ Frontend type check (`tsc --noEmit`) -4. ⚛️ ESLint -5. 🧪 Frontend unit tests (Vitest) -6. ⚛️ Next.js production build -7. 📏 PR size gate (warns > 1000 lines) - -### Environment Configuration - -Configuration uses **pydantic-settings** (v2) loaded from environment variables -with an optional `.env` file. - -**Key configuration groups (`backend/app/config.py`):** - -```python -class Settings(BaseSettings): - # App - APP_NAME: str = "Document AI Analyst" - ENVIRONMENT: str = "development" - SECRET_KEY: str # Required — change in production - - # Database - DATABASE_URL: str = "sqlite:///./data/app.db" - - # Auth - JWT_ALGORITHM: str = "HS256" - JWT_ACCESS_EXPIRY_MINUTES: int = 15 - JWT_REFRESH_EXPIRY_DAYS: int = 7 - GOOGLE_CLIENT_ID: str = "" # For Google OAuth - - # RAG Pipeline - CHUNK_SIZE: int = 1000 - CHUNK_OVERLAP: int = 200 - TOP_K_RETRIEVAL: int = 10 - TOP_K_RERANK: int = 5 - - # Embeddings - EMBEDDING_MODEL: str = "sentence-transformers/all-MiniLM-L6-v2" - - # LLM (HuggingFace) - HF_TOKEN: str # Required for Inference API - LLM_MODEL: str = "Qwen/Qwen2.5-72B-Instruct" - - # Celery / Redis - CELERY_BROKER_URL: str = "redis://localhost:6379/0" - - # File Upload - UPLOAD_DIR: str = "./data/uploads" - MAX_UPLOAD_SIZE_MB: int = 50 - - # ChromaDB - CHROMA_PERSIST_DIR: str = "./data/chroma_db" -``` - -**Environment files:** -- `.env.example` — Template with all variables and placeholder values -- `.env` — Local overrides (gitignored) -- **Never commit `.env` files with real secrets** - -**CORS configuration:** -- In production: restricted to `ALLOWED_ORIGINS` (comma-separated list) -- In development: open (`["*"]`) for local testing - ---- - -## Data Flow Diagrams - -### Upload → Process → Query +## Document Ingestion Flow ```mermaid sequenceDiagram @@ -808,121 +57,51 @@ sequenceDiagram participant Files as Upload storage participant Vector as ChromaDB - UI->>API: POST /api/v1/documents/upload (multipart file) + UI->>API: POST /api/v1/documents/upload API->>API: Validate filename, extension, size, MIME, and parser readability - API->>Files: Persist original file to {UPLOAD_DIR}/{user_id}/{filename} - API->>DB: Create Document row (status: pending) + API->>Files: Persist original file under the user's upload directory + API->>DB: Create document row with processing status API->>Redis: Queue Celery ingestion task - API-->>UI: 202 Accepted with document metadata + task_id - + API-->>UI: 202 Accepted with document metadata and task_id Redis->>Worker: Deliver ingestion task Worker->>Files: Read saved document - Worker->>Worker: Extract text & tables (pymupdf4llm / python-docx) - Worker->>Worker: Chunk text (recursive character splitter) - Worker->>Worker: Build knowledge graph (entity extraction + relationships) - Worker->>Worker: Generate summary (LLM) - Worker->>Vector: Store chunks with document_id + user_id metadata - Worker->>DB: Update Document row (status: ready, page_count, chunk_count, summary) - - UI->>API: GET /api/v1/documents/{id}/status?task_id=xxx - API-->>UI: { status: "ready", page_count: 12, chunk_count: 45 } - - UI->>API: GET /api/v1/documents/{id}/pdf - API-->>UI: PDF binary (or SSE-based page render) + Worker->>Worker: Extract pages, chunk text, build graph summary data + Worker->>Vector: Store chunks with document and user metadata + Worker->>DB: Save page count, chunk count, summary, and ready/failed status ``` -### Login → Token Refresh → OAuth +The upload route is intentionally strict before it writes long-lived state: +extension checks, size checks, MIME checks, and parser checks happen before the +file is moved into permanent storage. Celery uses Redis as the broker/result +backend, and the worker owns expensive work such as text extraction, chunking, +embedding, graph building, and summary generation. -```mermaid -sequenceDiagram - participant User as User - participant API as FastAPI /api/v1/auth - participant DB as SQL Database - participant Google as Google OAuth - - %% Password Login - User->>API: POST /auth/login { email, password } - API->>DB: Find user by email - API->>API: bcrypt.checkpw(password, user.hashed_password) - alt Invalid credentials - API-->>User: 401 Unauthorized - else Success - API->>API: create_access_token (15 min) - API->>API: create_refresh_token (7 days) - API-->>User: { access_token, refresh_token, user } - end - - %% Token Refresh - Note over User,API: 14 minutes later — access_token expires - User->>API: GET /api/v1/documents (with expired access_token) - API-->>User: 401 Unauthorized - User->>API: POST /auth/refresh { refresh_token } - API->>API: decode_token(token, "refresh") - alt Valid refresh token - API->>API: Issue new access_token (and optional new refresh_token) - API-->>User: { access_token, refresh_token? } - User->>API: Retry original request with new access_token - API-->>User: 200 OK - else Expired/invalid refresh token - API-->>User: 401 → User must re-login - end - - %% Google OAuth - User->>API: POST /auth/google { id_token } - API->>Google: Verify id_token via Google API (httpx) - Google-->>API: { email, name, sub, ... } - API->>DB: Find or create user by email - alt New user - API->>DB: Create User with is_verified=true (Google-verified) - end - API->>API: Create JWT tokens - API-->>User: { access_token, refresh_token, user } -``` - -### Question → RAG → Streamed Answer +## Chat And Retrieval Flow ```mermaid sequenceDiagram - participant UI as Chat Panel + participant UI as Frontend chat panel participant API as FastAPI chat route - participant DB as SQL Database - participant Retriever as Hybrid Retriever + participant DB as SQL chat/session rows + participant Retriever as Retriever and reranker participant Vector as ChromaDB - participant LLM as HuggingFace API + participant LLM as HuggingFace model - UI->>API: POST /api/v1/chat/ask/stream { question, session_id, document_id? } - API->>DB: Validate user, session, and optional document scope + UI->>API: POST /api/v1/chat/ask or /ask/stream + API->>DB: Validate user, optional document, and chat session API->>DB: Save user message - API->>API: Embed query (all-MiniLM-L6-v2 → 384-dim vector) - API->>Vector: Hybrid search (vector + BM25) with user/document filter - Vector-->>API: Top 10 candidate chunks - - API->>API: Cross-encoder reranking (ms-marco-MiniLM-L6-v2) - API->>API: Select top 5 chunks - - Note over API: Optional: GraphRAG traversal for entity relationships - - API->>API: Build prompt with selected chunks + conversation history - API->>LLM: POST HuggingFace Inference API (Qwen2.5-72B-Instruct) - LLM-->>API: SSE stream of answer tokens - - loop For each token - API-->>UI: data: { type: "token", data: "The" } - API-->>UI: data: { type: "token", data: " answer" } - API-->>UI: data: { type: "token", data: " is" } - UI->>UI: Append token to streaming message - end - - API->>LLM: (streaming completes) - LLM-->>API: Generation complete - API->>API: Collect full answer + source citations - API->>DB: Save assistant message with sources - API-->>UI: data: { type: "done", data: { message_id, sources: [...] } } - - UI->>UI: Finalize message with sources in SourceCard + API->>Retriever: Generate answer for question and optional document scope + Retriever->>Vector: Semantic search by user and document metadata + Retriever->>Retriever: Rerank candidate chunks + Retriever->>LLM: Send prompt with selected context + LLM-->>API: Answer tokens or complete answer + API->>DB: Save assistant response and source citations + API-->>UI: JSON response or server-sent events ``` ---- +Non-streaming chat returns a complete `ChatResponse`. Streaming chat uses +server-sent events so the frontend can render tokens as they arrive, then saves +the final assistant message after generation finishes. ## Data Ownership And Boundaries @@ -950,12 +129,6 @@ documents, chat sessions, messages, uploaded files, or vector chunks. Admin routes use `get_current_admin` and should avoid returning secrets, tokens, file contents, or raw vector payloads. -**Vector data isolation:** ChromaDB collections use a shared collection with -per-document `user_id` metadata. Every vector query filters by `user_id` to -prevent cross-user data leakage. - ---- - ## Swagger And OpenAPI Notes FastAPI builds the OpenAPI schema from route decorators, response models, @@ -969,8 +142,6 @@ an endpoint: - Mention asynchronous side effects, such as background ingestion or SSE streaming, in the route description. ---- - ## Local Contributor Checklist Before opening a backend documentation or route metadata PR: @@ -979,5 +150,3 @@ Before opening a backend documentation or route metadata PR: 2. Run the fatal-error flake8 selection used by CI. 3. Check Markdown fences and Mermaid blocks render as plain GitHub Markdown. 4. Confirm the README links to any new contributor-facing docs. -5. Run `npm test` in `frontend/` if touching frontend code. -6. Verify all CI checks pass before requesting review. diff --git a/docs/DATABASE.md b/docs/DATABASE.md deleted file mode 100644 index 38e32032..00000000 --- a/docs/DATABASE.md +++ /dev/null @@ -1,309 +0,0 @@ -# Database Schema - -This guide documents the backend relational schema used by -PDF-Assistant-RAG. The current implementation is defined with SQLAlchemy ORM -models in `backend/app/models.py` and is initialized through -`backend/app/database.py`. - -## Runtime Database - -The application reads `DATABASE_URL` from settings. SQLite is the default local -database, while non-SQLite URLs use SQLAlchemy's pooled engine configuration. -On startup, `init_db()` imports the models, creates any missing tables, and runs -small non-destructive migrations for columns added after existing databases were -created. - -All primary application tables use user-owned data boundaries. Most user data is -deleted through ORM cascades when a user row is removed. - -## Entity Relationship Diagram - -```mermaid -erDiagram - users ||--o{ documents : owns - users ||--o{ chat_messages : writes - users ||--o{ chat_sessions : starts - users ||--o{ api_keys : creates - users ||--o{ drive_connections : connects - users ||--o{ workspace_invitations : sends - - documents ||--o{ chat_messages : scopes - chat_sessions ||--o{ chat_messages : groups - chat_messages ||--o| shared_messages : exposes - - users { - uuid id PK - string username UK - string email UK - string hashed_password - enum role - boolean is_admin - boolean is_verified - datetime created_at - datetime last_login - } - - documents { - uuid id PK - uuid user_id FK - string filename - string original_name - integer file_size - integer page_count - integer chunk_count - string status - boolean is_deleted - datetime uploaded_at - datetime completed_at - } - - chat_sessions { - uuid id PK - uuid user_id FK - string title - datetime created_at - } - - chat_messages { - uuid id PK - uuid user_id FK - uuid document_id FK - uuid session_id FK - string role - text content - text sources_json - string feedback - datetime created_at - } - - api_keys { - uuid id PK - uuid user_id FK - string name - string key_prefix - string hashed_key UK - boolean is_active - datetime created_at - datetime last_used_at - } - - drive_connections { - uuid id PK - uuid user_id FK - string folder_id - boolean enabled - datetime last_synced_at - datetime created_at - datetime updated_at - } - - workspace_invitations { - string id PK - string email - string token_hash UK - string inviter_id FK - string workspace_name - datetime created_at - datetime expires_at - datetime accepted_at - } - - shared_messages { - uuid id PK - uuid message_id FK - datetime created_at - } -``` - -## Tables - -### `users` - -Stores registered users and authentication metadata. - -Important columns: - -- `id`: GUID primary key. -- `username` and `email`: unique indexed identity fields. -- `hashed_password`: password hash used for email/password login. -- `google_refresh_token` and `hf_token`: encrypted token fields. -- `role`: `user` or `admin` enum for role-based access control. -- `is_admin`: legacy admin flag kept alongside `role`. -- `is_verified`, `verification_token_hash`, - `verification_token_created_at`: email verification state. -- `created_at` and `last_login`: account audit timestamps. - -Relationships: - -- One user owns many `documents`. -- One user writes many `chat_messages`. -- One user starts many `chat_sessions`. -- One user owns many `api_keys`. -- One user owns many `drive_connections`. - -### `documents` - -Stores uploaded document metadata and ingestion status. File bytes live outside -the relational database in upload storage; vector chunks live in ChromaDB. - -Important columns: - -- `id`: GUID primary key. -- `user_id`: required foreign key to `users.id`. -- `filename` and `original_name`: stored filename and original upload name. -- `file_size`, `page_count`, `chunk_count`: processing metrics. -- `status`, `processing_progress`, `processing_stage`, `retry_count`: - ingestion state. -- `summary`, `chunk_size`, `chunk_overlap`, `extracted_urls`: document analysis - metadata. -- `drive_file_id`, `drive_folder_id`, `drive_synced_at`: Google Drive sync - metadata. -- `is_deleted` and `deleted_at`: soft-delete state. -- `uploaded_at`, `last_accessed_at`, `processing_started_at`, `completed_at`: - lifecycle timestamps. - -Relationships: - -- Many documents belong to one `users` row. -- One document can scope many `chat_messages`. - -### `chat_sessions` - -Groups chat messages into logical threads for a user. - -Important columns: - -- `id`: GUID primary key. -- `user_id`: required foreign key to `users.id`. -- `title`: user-facing session title. -- `created_at`: session creation timestamp. - -Relationships: - -- Many sessions belong to one user. -- One session groups many chat messages. - -### `chat_messages` - -Stores persistent chat history for user and assistant turns. - -Important columns: - -- `id`: GUID primary key. -- `user_id`: required foreign key to `users.id`. -- `document_id`: optional foreign key to `documents.id`; null allows - document-independent chat. -- `session_id`: optional foreign key to `chat_sessions.id`. -- `role`: message author role, commonly `user` or `assistant`. -- `content`: message body. -- `sources_json`: serialized source citation metadata. -- `feedback`: optional user feedback such as `up` or `down`. -- `created_at`: message timestamp. - -Relationships: - -- Many messages belong to one user. -- Many messages can be scoped to one document. -- Many messages can be grouped under one chat session. -- One message can have one `shared_messages` row. - -### `api_keys` - -Stores hashed API keys for programmatic access. - -Important columns: - -- `id`: GUID primary key. -- `user_id`: required foreign key to `users.id`. -- `name`: user-facing key name. -- `key_prefix`: short visible prefix for identification. -- `hashed_key`: unique indexed key hash. -- `is_active`: revocation flag. -- `created_at` and `last_used_at`: audit timestamps. - -Relationships: - -- Many API keys belong to one user. - -### `drive_connections` - -Stores Google Drive sync connection metadata. - -Important columns: - -- `id`: GUID primary key. -- `user_id`: required foreign key to `users.id`. -- `folder_id`: connected Drive folder. -- `credentials_json` and `service_account_file`: credential references. -- `enabled`: sync toggle. -- `last_synced_at`, `created_at`, `updated_at`: sync lifecycle timestamps. - -Relationships: - -- Many Drive connections belong to one user. - -### `workspace_invitations` - -Stores pending workspace invitations. - -Important columns: - -- `id`: string primary key generated with `uuid4`. -- `email`: invited email address. -- `token_hash`: unique indexed invitation token hash. -- `inviter_id`: required foreign key to `users.id`. -- `workspace_name`: target workspace display name. -- `created_at`, `expires_at`, `accepted_at`: invitation lifecycle timestamps. - -Relationships: - -- Many invitations can be sent by one user. - -### `shared_messages` - -Links chat messages to public share records. - -Important columns: - -- `id`: GUID primary key. -- `message_id`: required unique foreign key to `chat_messages.id`. -- `created_at`: share creation timestamp. - -Relationships: - -- One shared message belongs to exactly one chat message. - -## Relationship Summary - -| Parent | Child | Cardinality | Delete behavior | -| --- | --- | --- | --- | -| `users` | `documents` | one-to-many | ORM cascade delete-orphan | -| `users` | `chat_messages` | one-to-many | ORM cascade delete-orphan | -| `users` | `chat_sessions` | one-to-many | ORM cascade delete-orphan | -| `users` | `api_keys` | one-to-many | ORM cascade delete-orphan | -| `users` | `drive_connections` | one-to-many | ORM cascade delete-orphan | -| `users` | `workspace_invitations` | one-to-many | no explicit cascade | -| `documents` | `chat_messages` | one-to-many | ORM cascade delete-orphan | -| `chat_sessions` | `chat_messages` | one-to-many | ORM cascade delete-orphan | -| `chat_messages` | `shared_messages` | one-to-one | ORM cascade delete-orphan | - -## Data Ownership Rules - -- User-facing document and chat queries should filter by `user_id`. -- Document-scoped chat messages should validate that the document belongs to - the authenticated user before reading or writing history. -- Shared messages expose one selected chat message and should not bypass other - ownership checks when adding new sharing features. -- Admin routes should aggregate operational data without returning encrypted - tokens, password hashes, raw file contents, or vector payloads. - -## Migration Notes - -`Base.metadata.create_all()` creates missing tables, but it does not add new -columns to existing tables. The `_migrate_schema()` helper in -`backend/app/database.py` applies small SQLite-compatible column additions for -existing databases, including newer user verification fields, API key metadata, -document soft-delete and Drive metadata, and chat message feedback. - -For larger schema changes, prefer an explicit migration plan instead of relying -on startup-time column checks. diff --git a/frontend/.dockerignore b/frontend/.dockerignore deleted file mode 100644 index 3fbb4b39..00000000 --- a/frontend/.dockerignore +++ /dev/null @@ -1,33 +0,0 @@ -# Dependencies — reinstalled inside Docker -node_modules -.pnp -.pnp.js - -# Next.js build output (rebuilt inside Docker) -.next -out - -# Dev / test artifacts -coverage -.nyc_output -*.test.* -*.spec.* -__tests__ -playwright-report -test-results - -# Environment files — never bake into image -.env -.env.* -!.env.example - -# Editor / OS artifacts -.DS_Store -.vscode -.idea -*.log -Thumbs.db - -# Git -.git -.gitignore diff --git a/frontend/Dockerfile b/frontend/Dockerfile deleted file mode 100644 index e737111d..00000000 --- a/frontend/Dockerfile +++ /dev/null @@ -1,89 +0,0 @@ -# syntax=docker/dockerfile:1 -# ============================================================================= -# Multi-stage Dockerfile for the PDF-Assistant-RAG Next.js frontend. -# -# next.config.ts uses output: "export" which generates a fully static site -# under /out — no Node.js runtime is needed at serve time. The final image -# uses nginx:alpine to serve the static assets, eliminating all node_modules -# and dev dependencies from the production image. -# -# Stages: -# deps — install production + dev deps from lockfile (cached layer) -# builder — run `next build` to produce the static /out directory -# runner — nginx:alpine serving /out; no Node.js, no node_modules -# ============================================================================= - -# ----------------------------------------------------------------------------- -# Stage 1 — deps: install dependencies from lockfile -# ----------------------------------------------------------------------------- -FROM node:20-alpine AS deps - -# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine -# for why libc6-compat may be needed. -RUN apk add --no-cache libc6-compat - -WORKDIR /app - -# Copy only the manifests first so this layer is cached unless deps change -COPY package.json package-lock.json ./ -RUN npm ci --no-audit --no-fund - -# ----------------------------------------------------------------------------- -# Stage 2 — builder: compile the Next.js static export -# ----------------------------------------------------------------------------- -FROM node:20-alpine AS builder - -RUN apk add --no-cache libc6-compat - -WORKDIR /app - -# Restore installed modules from deps stage -COPY --from=deps /app/node_modules ./node_modules - -# Copy full source -COPY . . - -# next build reads NEXT_PUBLIC_* vars at build time — pass them as ARGs so -# CI/CD pipelines can inject the correct API URL without baking it into the -# image unconditionally. -ARG NEXT_PUBLIC_API_URL=http://localhost:7860 -ENV NEXT_PUBLIC_API_URL=${NEXT_PUBLIC_API_URL} - -# Disable Next.js telemetry during build -ENV NEXT_TELEMETRY_DISABLED=1 - -# Produce the static export in /app/out -RUN npm run build - -# ----------------------------------------------------------------------------- -# Stage 3 — runner: nginx serves the static export -# No Node.js, no node_modules, no source code in this image. -# ----------------------------------------------------------------------------- -FROM nginx:1.27-alpine AS runner - -# Remove the default nginx welcome page -RUN rm -rf /usr/share/nginx/html/* - -# Copy compiled static assets from builder -COPY --from=builder /app/out /usr/share/nginx/html - -# Custom nginx config: -# - try_files for SPA-style client-side routing fallback -# - gzip compression for JS/CSS/HTML -# - cache headers for static assets -COPY nginx.conf /etc/nginx/conf.d/default.conf - -# nginx runs as root by default; drop to a non-root user for security -RUN addgroup -g 1001 -S appgroup && \ - adduser -u 1001 -S appuser -G appgroup && \ - chown -R appuser:appgroup /usr/share/nginx/html && \ - chown -R appuser:appgroup /var/cache/nginx && \ - chown -R appuser:appgroup /var/log/nginx && \ - touch /var/run/nginx.pid && \ - chown appuser:appgroup /var/run/nginx.pid - -USER appuser - -EXPOSE 3000 - -CMD ["nginx", "-g", "daemon off;"] diff --git a/frontend/e2e/auth-and-chat.spec.ts b/frontend/e2e/auth-and-chat.spec.ts index aeb95ce8..6ed0db17 100644 --- a/frontend/e2e/auth-and-chat.spec.ts +++ b/frontend/e2e/auth-and-chat.spec.ts @@ -48,12 +48,6 @@ async function mockDashboardApis(page: Page, documents: typeof uploadedDocument[ }, }); }); - - await page.route("**/api/v1/chat/sessions", async (route) => { - await route.fulfill({ - json: [], - }); - }); } test("logs in with email and password", async ({ page }) => { @@ -67,10 +61,9 @@ test("logs in with email and password", async ({ page }) => { await page.goto("/login"); await page.locator("#login-email").fill(user.email); await page.locator("#login-password").fill("password123"); - await Promise.all([ - page.waitForURL("/dashboard"), - page.locator("#sign-in-btn").click(), - ]); + await page.locator("#sign-in-btn").click(); + + await expect(page).toHaveURL(/\/dashboard$/); await expect(page.getByText("No documents yet")).toBeVisible(); }); @@ -149,6 +142,7 @@ test("uploads a PDF document and chats with it", async ({ page }) => { await page.goto("/dashboard"); + // Upload as a PDF await page.locator('input[type="file"]').setInputFiles({ name: "test.pdf", mimeType: "application/pdf", @@ -190,9 +184,12 @@ test("deletes a document successfully", async ({ page }) => { const documentButton = page.getByRole("button", { name: /test\.pdf/ }); await expect(documentButton).toBeVisible(); + // Handle confirm dialog (must be registered BEFORE click) page.on('dialog', dialog => dialog.accept()); + // Delete the document await documentButton.hover(); + // Find the button with Trash2 icon await page.locator('button.shrink-0:has(svg.lucide-trash2)').click(); await expect(page.getByText("No documents yet")).toBeVisible(); @@ -213,4 +210,4 @@ test("logs out successfully", async ({ page }) => { await expect(page).toHaveURL(/\/login$/); const token = await page.evaluate(() => localStorage.getItem("token")); expect(token).toBeNull(); -}); \ No newline at end of file +}); diff --git a/frontend/e2e/huggingface-token.spec.ts b/frontend/e2e/huggingface-token.spec.ts deleted file mode 100644 index ac2c87f3..00000000 --- a/frontend/e2e/huggingface-token.spec.ts +++ /dev/null @@ -1,298 +0,0 @@ -import { expect, test, type Page } from "@playwright/test"; - -// ── Shared fixtures ──────────────────────────────────────────────────────────── - -const user = { - id: "user-1", - username: "tester", - email: "tester@example.com", - is_admin: false, - is_verified: true, - role: "user", - created_at: "2026-01-01T00:00:00Z", -}; - -const userWithToken = { - ...user, - hf_token: "hf_existingToken1234567890", -}; - -// ── Helpers ──────────────────────────────────────────────────────────────────── - -/** Seed localStorage so the app boots as logged-in */ -async function seedAuth(page: Page, withHfToken = false) { - await page.addInitScript( - ({ withHfToken }) => { - localStorage.setItem("token", "access-token"); - localStorage.setItem("refresh_token", "refresh-token"); - if (withHfToken) { - // Simulate a session where the user already has a token stored - (window as unknown as Record).__hf_token_preset__ = - true; - } - }, - { withHfToken }, - ); -} - -/** Mock all APIs needed for the dashboard + settings page to load */ -async function mockBaseApis(page: Page, currentUser = user) { - await page.route("**/api/v1/auth/me", (route) => - route.fulfill({ json: currentUser }), - ); - await page.route("**/api/v1/documents/", (route) => - route.fulfill({ - json: { - items: [], - total: 0, - page: 1, - pages: 0, - total_pages: 0, - limit: 20, - }, - }), - ); - await page.route("**/api/v1/chat/sessions", (route) => - route.fulfill({ json: [] }), - ); -} - -/** Open the HuggingFace Token modal via the user-menu on the dashboard */ -async function openHfModal(page: Page) { - await page.getByRole("button", { name: user.username }).click(); - await page.getByRole("menuitem", { name: /huggingface token/i }).click(); - await expect(page.getByRole("dialog")).toBeVisible(); -} - -// ── Tests ────────────────────────────────────────────────────────────────────── - -test.describe("HuggingFace Token Flow", () => { - test("saves a valid token and stores it encrypted via the API", async ({ - page, - }) => { - let capturedBody: Record | null = null; - - await seedAuth(page); - await mockBaseApis(page); - - // The PUT /hf-token endpoint — assert payload and return updated user - await page.route("**/api/v1/auth/hf-token", async (route) => { - expect(route.request().method()).toBe("PUT"); - capturedBody = route.request().postDataJSON() as Record; - await route.fulfill({ - json: { ...user, hf_token: capturedBody.hf_token }, - }); - }); - - await page.goto("/dashboard"); - await openHfModal(page); - - // Fill a valid token - await page - .getByLabel("HuggingFace API Token") - .fill("hf_validTokenForTesting1234"); - await page.getByRole("button", { name: "Save Token" }).click(); - - // Success banner visible - await expect(page.getByText("Token saved successfully")).toBeVisible(); - - // API was called with the correct field name and token value - expect(capturedBody).not.toBeNull(); - expect(capturedBody!.hf_token).toBe("hf_validTokenForTesting1234"); - }); - - test("shows a validation error for a token that does not start with hf_", async ({ - page, - }) => { - await seedAuth(page); - await mockBaseApis(page); - - await page.goto("/dashboard"); - await openHfModal(page); - - await page.getByLabel("HuggingFace API Token").fill("sk-invalidToken12345"); - await page.getByRole("button", { name: "Save Token" }).click(); - - await expect(page.getByText("Token must start with 'hf_'")).toBeVisible(); - // Dialog stays open — no API call was made - await expect(page.getByRole("dialog")).toBeVisible(); - }); - - test("shows a validation error for a token that is too short", async ({ - page, - }) => { - await seedAuth(page); - await mockBaseApis(page); - - await page.goto("/dashboard"); - await openHfModal(page); - - await page.getByLabel("HuggingFace API Token").fill("hf_short"); - await page.getByRole("button", { name: "Save Token" }).click(); - - await expect(page.getByText(/too short/i)).toBeVisible(); - await expect(page.getByRole("dialog")).toBeVisible(); - }); - - test("shows a validation error when the input is empty", async ({ page }) => { - await seedAuth(page); - await mockBaseApis(page); - - await page.goto("/dashboard"); - await openHfModal(page); - - // Save button must be disabled when input is empty - await expect( - page.getByRole("button", { name: "Save Token" }), - ).toBeDisabled(); - }); - - test("shows an API error when the backend rejects the token", async ({ - page, - }) => { - await seedAuth(page); - await mockBaseApis(page); - - await page.route("**/api/v1/auth/hf-token", async (route) => { - await route.fulfill({ - status: 400, - json: { detail: "Invalid HuggingFace token" }, - }); - }); - - await page.goto("/dashboard"); - await openHfModal(page); - - await page - .getByLabel("HuggingFace API Token") - .fill("hf_validLengthButRejected1234"); - await page.getByRole("button", { name: "Save Token" }).click(); - - await expect(page.getByText("Invalid HuggingFace token")).toBeVisible(); - // Dialog stays open so the user can correct the token - await expect(page.getByRole("dialog")).toBeVisible(); - }); - - test("toggles token visibility with the show/hide button", async ({ - page, - }) => { - await seedAuth(page); - await mockBaseApis(page); - - await page.goto("/dashboard"); - await openHfModal(page); - - const input = page.getByLabel("HuggingFace API Token"); - await input.fill("hf_validTokenForTesting1234"); - - // Default — masked - await expect(input).toHaveAttribute("type", "password"); - - // Show - await page.getByRole("button", { name: "Show token" }).click(); - await expect(input).toHaveAttribute("type", "text"); - - // Hide again - await page.getByRole("button", { name: "Hide token" }).click(); - await expect(input).toHaveAttribute("type", "password"); - }); - - test("displays existing token preview when user already has a token", async ({ - page, - }) => { - await seedAuth(page, true); - // Return the user with a pre-existing token - await mockBaseApis(page, userWithToken); - - await page.goto("/dashboard"); - await openHfModal(page); - - // Modal shows the "Token configured" badge - await expect(page.getByText("Token configured")).toBeVisible(); - // Shows the masked preview: first 7 chars + **** + last 4 - await expect( - page.getByText(/hf_exis\*{4}\d{4}|hf_exis\*{4}7890/), - ).toBeVisible(); - // Save button says "Update Token" for existing tokens - await expect( - page.getByRole("button", { name: "Update Token" }), - ).toBeVisible(); - }); - - test("removes an existing token successfully", async ({ page }) => { - let removeCalled = false; - - await seedAuth(page, true); - await mockBaseApis(page, userWithToken); - - await page.route("**/api/v1/auth/hf-token", async (route) => { - expect(route.request().method()).toBe("PUT"); - const body = route.request().postDataJSON() as Record; - expect(body.hf_token).toBe(""); - removeCalled = true; - await route.fulfill({ - json: { ...user, hf_token: "" }, - }); - }); - - await page.goto("/dashboard"); - await openHfModal(page); - - await page.getByRole("button", { name: /remove/i }).click(); - - await expect(page.getByText("Token removed successfully")).toBeVisible(); - - expect(removeCalled).toBe(true); - }); - - test("cancel button closes the modal without making any API call", async ({ - page, - }) => { - let apiCalled = false; - - await seedAuth(page); - await mockBaseApis(page); - - await page.route("**/api/v1/auth/hf-token", async (route) => { - apiCalled = true; - await route.fulfill({ json: user }); - }); - - await page.goto("/dashboard"); - await openHfModal(page); - - await page - .getByLabel("HuggingFace API Token") - .fill("hf_someToken1234567890"); - await page.getByRole("button", { name: "Cancel" }).click(); - - await expect(page.getByRole("dialog")).not.toBeVisible(); - expect(apiCalled).toBe(false); - }); - - test("updates an existing token with a new valid token", async ({ page }) => { - let capturedBody: Record | null = null; - - await seedAuth(page, true); - await mockBaseApis(page, userWithToken); - - await page.route("**/api/v1/auth/hf-token", async (route) => { - capturedBody = route.request().postDataJSON() as Record; - await route.fulfill({ - json: { ...user, hf_token: capturedBody.hf_token }, - }); - }); - - await page.goto("/dashboard"); - await openHfModal(page); - - // Clear and type a new token - await page - .getByLabel("HuggingFace API Token") - .fill("hf_newReplacementToken12345"); - await page.getByRole("button", { name: "Update Token" }).click(); - - await expect(page.getByText("Token saved successfully")).toBeVisible(); - expect(capturedBody!.hf_token).toBe("hf_newReplacementToken12345"); - }); -}); diff --git a/frontend/e2e/snapshots.spec.ts b/frontend/e2e/snapshots.spec.ts index ad16f268..7d8b7b3d 100644 --- a/frontend/e2e/snapshots.spec.ts +++ b/frontend/e2e/snapshots.spec.ts @@ -1,23 +1,10 @@ -/** - * Visual regression tests for the landing page and key UI surfaces. - * - * Snapshots are stored in e2e/snapshots.spec.ts-snapshots/ and committed - * to the repository so CI can diff against them on every PR. - * - * To regenerate baselines (e.g. after an intentional UI change): - * npx playwright test snapshots.spec.ts --update-snapshots - */ import { expect, test, type Page } from "@playwright/test"; -// ── Shared fixtures ─────────────────────────────────────────────────────────── - const user = { id: "user-1", username: "tester", email: "tester@example.com", - is_verified: true, is_admin: false, - role: "user", created_at: "2026-05-28T00:00:00Z", }; @@ -32,17 +19,11 @@ const uploadedDocument = { uploaded_at: "2026-05-28T00:00:00Z", }; -async function mockAuthApis(page: Page) { +async function mockDashboardApis(page: Page, documents: typeof uploadedDocument[] = []) { await page.route("**/api/v1/auth/me", async (route) => { await route.fulfill({ json: user }); }); -} -async function mockDashboardApis( - page: Page, - documents: typeof uploadedDocument[] = [] -) { - await mockAuthApis(page); await page.route("**/api/v1/documents/", async (route) => { await route.fulfill({ json: { @@ -50,146 +31,79 @@ async function mockDashboardApis( total: documents.length, page: 1, pages: documents.length > 0 ? 1 : 0, - total_pages: documents.length > 0 ? 1 : 0, - limit: 20, }, }); }); - - await page.route("**/api/v1/chat/sessions", async (route) => { - await route.fulfill({ json: [] }); - }); -} - -/** Wait for network and CSS animations to settle before snapshotting. */ -async function stabilise(page: Page) { - await page.waitForLoadState("networkidle"); - // One rAF to let React finish any pending paint - await page.evaluate(() => new Promise((r) => requestAnimationFrame(r))); } -// ── Landing page ────────────────────────────────────────────────────────────── - -test.describe("Landing page visual regression", () => { - test.beforeEach(async ({ page }) => { - // Ensure unauthenticated state — no redirect to /dashboard - await page.route("**/api/v1/auth/me", async (route) => { - await route.fulfill({ status: 401, json: { detail: "Not authenticated" } }); - }); - }); - - test("landing page — full page", async ({ page }) => { - await page.goto("/"); - await stabilise(page); - - await expect( - page.getByRole("heading", { name: /chat with your/i }) - ).toBeVisible(); - - await expect(page).toHaveScreenshot("landing-full.png", { - fullPage: true, - }); - }); - - test("landing page — hero section", async ({ page }) => { - await page.goto("/"); - await stabilise(page); - - const hero = page.locator("section").first(); - await expect(hero).toHaveScreenshot("landing-hero.png"); - }); - - test("landing page — feature cards grid", async ({ page }) => { - await page.goto("/"); - await stabilise(page); - - // The feature cards grid is the last element inside the hero section - const grid = page.locator("section .grid").first(); - await expect(grid).toHaveScreenshot("landing-feature-cards.png"); - }); - - test("landing page — footer", async ({ page }) => { - await page.goto("/"); - await stabilise(page); - - const footer = page.locator("footer"); - await expect(footer).toHaveScreenshot("landing-footer.png"); - }); - - test("landing page — CTA buttons visible", async ({ page }) => { - await page.goto("/"); - await stabilise(page); - - await expect( - page.getByRole("link", { name: "Get Started Free" }) - ).toBeVisible(); - await expect(page.getByRole("link", { name: "Sign In" })).toBeVisible(); - await expect( - page.getByRole("link", { name: "Developer API" }) - ).toBeVisible(); - }); -}); - -// ── Auth pages ──────────────────────────────────────────────────────────────── - -test.describe("Auth pages visual regression", () => { - test("login page — full page", async ({ page }) => { - await page.goto("/login"); - await page.waitForSelector("#login-email"); - await stabilise(page); - - await expect(page).toHaveScreenshot("login-full.png", { fullPage: true }); - }); - - test("login page — form", async ({ page }) => { +test.describe("Frontend Snapshot Tests", () => { + test("login page snapshot", async ({ page }) => { await page.goto("/login"); await page.waitForSelector("#login-email"); - await stabilise(page); - - const form = page.locator("form").first(); - await expect(form).toHaveScreenshot("login-form.png"); + + if (!process.env.CI) { + await expect(page).toHaveScreenshot("login-page.png", { + maxDiffPixelRatio: 0.1, + threshold: 0.2, + }); + } else { + await expect(page.locator("#login-email")).toBeVisible(); + } }); - test("register page — full page", async ({ page }) => { + test("register page snapshot", async ({ page }) => { await page.goto("/register"); await page.waitForSelector("#reg-username"); - await stabilise(page); - - await expect(page).toHaveScreenshot("register-full.png", { - fullPage: true, - }); + + if (!process.env.CI) { + await expect(page).toHaveScreenshot("register-page.png", { + maxDiffPixelRatio: 0.1, + threshold: 0.2, + }); + } else { + await expect(page.locator("#reg-username")).toBeVisible(); + } }); -}); - -// ── Dashboard ───────────────────────────────────────────────────────────────── -test.describe("Dashboard visual regression", () => { - test.beforeEach(async ({ page }) => { + test("dashboard empty page snapshot", async ({ page }) => { + // Set mock token await page.addInitScript(() => { localStorage.setItem("token", "access-token"); localStorage.setItem("refresh_token", "refresh-token"); }); - }); - test("dashboard — empty state", async ({ page }) => { await mockDashboardApis(page, []); await page.goto("/dashboard"); await page.waitForSelector("text=No documents yet"); - await stabilise(page); - await expect(page).toHaveScreenshot("dashboard-empty.png", { - fullPage: true, - }); + if (!process.env.CI) { + await expect(page).toHaveScreenshot("dashboard-empty.png", { + maxDiffPixelRatio: 0.1, + threshold: 0.2, + }); + } else { + await expect(page.locator("text=No documents yet")).toBeVisible(); + } }); - test("dashboard — with one document", async ({ page }) => { + test("dashboard with document page snapshot", async ({ page }) => { + // Set mock token + await page.addInitScript(() => { + localStorage.setItem("token", "access-token"); + localStorage.setItem("refresh_token", "refresh-token"); + }); + await mockDashboardApis(page, [uploadedDocument]); await page.goto("/dashboard"); await page.waitForSelector("text=notes.txt"); - await stabilise(page); - await expect(page).toHaveScreenshot("dashboard-with-doc.png", { - fullPage: true, - }); + if (!process.env.CI) { + await expect(page).toHaveScreenshot("dashboard-with-doc.png", { + maxDiffPixelRatio: 0.1, + threshold: 0.2, + }); + } else { + await expect(page.locator("text=notes.txt")).toBeVisible(); + } }); }); diff --git a/frontend/e2e/snapshots.spec.ts-snapshots/dashboard-empty-chromium-win32.png b/frontend/e2e/snapshots.spec.ts-snapshots/dashboard-empty-chromium-win32.png deleted file mode 100644 index bc612894..00000000 --- a/frontend/e2e/snapshots.spec.ts-snapshots/dashboard-empty-chromium-win32.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7be08155a5c47360429d57a1a0e72c9a4c9217395c5e52355654fbbfffa257bc -size 48749 diff --git a/frontend/e2e/snapshots.spec.ts-snapshots/dashboard-with-doc-chromium-win32.png b/frontend/e2e/snapshots.spec.ts-snapshots/dashboard-with-doc-chromium-win32.png deleted file mode 100644 index ab1d28f2..00000000 --- a/frontend/e2e/snapshots.spec.ts-snapshots/dashboard-with-doc-chromium-win32.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bdde5f6da5cdc3b2a3955431637a46401cb42e96efb2f93f7dacfe31379a7f42 -size 48915 diff --git a/frontend/e2e/snapshots.spec.ts-snapshots/login-page-chromium-win32.png b/frontend/e2e/snapshots.spec.ts-snapshots/login-page-chromium-win32.png deleted file mode 100644 index c396450a..00000000 --- a/frontend/e2e/snapshots.spec.ts-snapshots/login-page-chromium-win32.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:972dc43ca6defb778760ba7cc52206af8da31b5c5c38985fdcd781f00baae546 -size 49599 diff --git a/frontend/e2e/snapshots.spec.ts-snapshots/register-page-chromium-win32.png b/frontend/e2e/snapshots.spec.ts-snapshots/register-page-chromium-win32.png deleted file mode 100644 index 3ae42ec3..00000000 --- a/frontend/e2e/snapshots.spec.ts-snapshots/register-page-chromium-win32.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ff4158e8f7e576fb1c2544c8bdbade3787bede6d16ec471b207316c805394258 -size 55879 diff --git a/frontend/nginx.conf b/frontend/nginx.conf deleted file mode 100644 index cdfcef4a..00000000 --- a/frontend/nginx.conf +++ /dev/null @@ -1,47 +0,0 @@ -server { - listen 3000; - server_name _; - - root /usr/share/nginx/html; - index index.html; - - # ── Gzip compression ────────────────────────────────────────────────── - gzip on; - gzip_comp_level 5; - gzip_min_length 256; - gzip_proxied any; - gzip_types - application/javascript - application/json - application/xml - text/css - text/html - text/plain - text/xml - image/svg+xml; - - # ── Static asset caching ────────────────────────────────────────────── - # Next.js hashes JS/CSS filenames — safe to cache aggressively - location /_next/static/ { - expires 1y; - add_header Cache-Control "public, immutable"; - access_log off; - } - - # ── SPA fallback routing ────────────────────────────────────────────── - # next export with output: "export" generates per-route .html files. - # Try the exact file, then the directory index, then fall back to - # index.html so client-side navigation works correctly. - location / { - try_files $uri $uri.html $uri/ /index.html; - } - - # ── Security headers ────────────────────────────────────────────────── - add_header X-Content-Type-Options "nosniff" always; - add_header X-Frame-Options "SAMEORIGIN" always; - add_header X-XSS-Protection "1; mode=block" always; - add_header Referrer-Policy "strict-origin-when-cross-origin" always; - add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; - add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'wasm-unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' https:; connect-src 'self' https:;" always; - add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always; -} \ No newline at end of file diff --git a/frontend/package-lock.json b/frontend/package-lock.json index c7583db9..6b74393e 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -10,7 +10,6 @@ "dependencies": { "@base-ui/react": "^1.4.1", "@tailwindcss/typography": "^0.5.19", - "@xyflow/react": "^12.11.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "date-fns": "^4.4.0", @@ -19,7 +18,7 @@ "lucide-react": "^1.8.0", "next": "16.2.4", "next-themes": "^0.4.6", - "pdfjs-dist": "5.4.296", + "pdfjs-dist": "^5.6.205", "react": "19.2.4", "react-dom": "19.2.4", "react-dropzone": "^15.0.0", @@ -2569,6 +2568,9 @@ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -2586,6 +2588,9 @@ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -2603,6 +2608,9 @@ "ppc64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -2620,6 +2628,9 @@ "s390x" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -2637,6 +2648,9 @@ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -2654,6 +2668,9 @@ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -3302,55 +3319,6 @@ "assertion-error": "^2.0.1" } }, - "node_modules/@types/d3-color": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz", - "integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==", - "license": "MIT" - }, - "node_modules/@types/d3-drag": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/@types/d3-drag/-/d3-drag-3.0.7.tgz", - "integrity": "sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ==", - "license": "MIT", - "dependencies": { - "@types/d3-selection": "*" - } - }, - "node_modules/@types/d3-interpolate": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz", - "integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==", - "license": "MIT", - "dependencies": { - "@types/d3-color": "*" - } - }, - "node_modules/@types/d3-selection": { - "version": "3.0.11", - "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-3.0.11.tgz", - "integrity": "sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w==", - "license": "MIT" - }, - "node_modules/@types/d3-transition": { - "version": "3.0.9", - "resolved": "https://registry.npmjs.org/@types/d3-transition/-/d3-transition-3.0.9.tgz", - "integrity": "sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg==", - "license": "MIT", - "dependencies": { - "@types/d3-selection": "*" - } - }, - "node_modules/@types/d3-zoom": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/@types/d3-zoom/-/d3-zoom-3.0.8.tgz", - "integrity": "sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw==", - "license": "MIT", - "dependencies": { - "@types/d3-interpolate": "*", - "@types/d3-selection": "*" - } - }, "node_modules/@types/debug": { "version": "4.1.13", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", @@ -3442,7 +3410,7 @@ "version": "19.2.3", "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.3.tgz", "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", - "devOptional": true, + "dev": true, "license": "MIT", "peerDependencies": { "@types/react": "^19.2.0" @@ -4184,76 +4152,6 @@ "url": "https://opencollective.com/vitest" } }, - "node_modules/@xyflow/react": { - "version": "12.11.0", - "resolved": "https://registry.npmjs.org/@xyflow/react/-/react-12.11.0.tgz", - "integrity": "sha512-na4IO33FSs2OS72hASgZDmTYwFAkef7Z74uBUVrong3ARmQQHfnRUVaCFn1kTt5LbS6pK03TbYjCPGLjLFfziA==", - "license": "MIT", - "dependencies": { - "@xyflow/system": "0.0.77", - "classcat": "^5.0.3", - "zustand": "^4.4.0" - }, - "peerDependencies": { - "@types/react": ">=17", - "@types/react-dom": ">=17", - "react": ">=17", - "react-dom": ">=17" - }, - "peerDependenciesMeta": { - "@types/react": { - "optional": true - }, - "@types/react-dom": { - "optional": true - } - } - }, - "node_modules/@xyflow/react/node_modules/zustand": { - "version": "4.5.7", - "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.5.7.tgz", - "integrity": "sha512-CHOUy7mu3lbD6o6LJLfllpjkzhHXSBlX8B9+qPddUsIfeF5S/UZ5q0kmCsnRqT1UHFQZchNFDDzMbQsuesHWlw==", - "license": "MIT", - "dependencies": { - "use-sync-external-store": "^1.2.2" - }, - "engines": { - "node": ">=12.7.0" - }, - "peerDependencies": { - "@types/react": ">=16.8", - "immer": ">=9.0.6", - "react": ">=16.8" - }, - "peerDependenciesMeta": { - "@types/react": { - "optional": true - }, - "immer": { - "optional": true - }, - "react": { - "optional": true - } - } - }, - "node_modules/@xyflow/system": { - "version": "0.0.77", - "resolved": "https://registry.npmjs.org/@xyflow/system/-/system-0.0.77.tgz", - "integrity": "sha512-qCDCMCQAAgUu8yHnhloHG9F5mwPX5E+Wl8McpYIOPSSXfzFJJoZcwOcsDiAjitVKIg2de1WmJbCHfpcvxprsgg==", - "license": "MIT", - "dependencies": { - "@types/d3-drag": "^3.0.7", - "@types/d3-interpolate": "^3.0.4", - "@types/d3-selection": "^3.0.10", - "@types/d3-transition": "^3.0.8", - "@types/d3-zoom": "^3.0.8", - "d3-drag": "^3.0.0", - "d3-interpolate": "^3.0.1", - "d3-selection": "^3.0.0", - "d3-zoom": "^3.0.0" - } - }, "node_modules/accepts": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", @@ -4951,12 +4849,6 @@ "url": "https://polar.sh/cva" } }, - "node_modules/classcat": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/classcat/-/classcat-5.0.5.tgz", - "integrity": "sha512-JhZUT7JFcQy/EzW605k/ktHtncoo9vnyW/2GspNYwFlN1C/WmjuV/xtS04e9SOkL2sTdw0VAZ2UGCcQ9lR6p6w==", - "license": "MIT" - }, "node_modules/cli-cursor": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-5.0.0.tgz", @@ -5255,111 +5147,6 @@ "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", "license": "MIT" }, - "node_modules/d3-color": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", - "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-dispatch": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", - "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==", - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-drag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-3.0.0.tgz", - "integrity": "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==", - "license": "ISC", - "dependencies": { - "d3-dispatch": "1 - 3", - "d3-selection": "3" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-ease": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", - "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", - "license": "BSD-3-Clause", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-interpolate": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", - "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", - "license": "ISC", - "dependencies": { - "d3-color": "1 - 3" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-selection": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", - "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-timer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", - "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/d3-transition": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-3.0.1.tgz", - "integrity": "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==", - "license": "ISC", - "dependencies": { - "d3-color": "1 - 3", - "d3-dispatch": "1 - 3", - "d3-ease": "1 - 3", - "d3-interpolate": "1 - 3", - "d3-timer": "1 - 3" - }, - "engines": { - "node": ">=12" - }, - "peerDependencies": { - "d3-selection": "2 - 3" - } - }, - "node_modules/d3-zoom": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-3.0.0.tgz", - "integrity": "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==", - "license": "ISC", - "dependencies": { - "d3-dispatch": "1 - 3", - "d3-drag": "2 - 3", - "d3-interpolate": "1 - 3", - "d3-selection": "2 - 3", - "d3-transition": "2 - 3" - }, - "engines": { - "node": ">=12" - } - }, "node_modules/damerau-levenshtein": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz", @@ -10135,6 +9922,13 @@ "url": "https://opencollective.com/node-fetch" } }, + "node_modules/node-readable-to-web-readable-stream": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/node-readable-to-web-readable-stream/-/node-readable-to-web-readable-stream-0.4.2.tgz", + "integrity": "sha512-/cMZNI34v//jUTrI+UIo4ieHAB5EZRY/+7OmXZgBxaWBMcW2tGdceIw06RFxWxrKZ5Jp3sI2i5TsRo+CBhtVLQ==", + "license": "MIT", + "optional": true + }, "node_modules/node-releases": { "version": "2.0.37", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.37.tgz", @@ -10610,15 +10404,16 @@ "license": "MIT" }, "node_modules/pdfjs-dist": { - "version": "5.4.296", - "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.296.tgz", - "integrity": "sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==", + "version": "5.6.205", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.6.205.tgz", + "integrity": "sha512-tlUj+2IDa7G1SbvBNN74UHRLJybZDWYom+k6p5KIZl7huBvsA4APi6mKL+zCxd3tLjN5hOOEE9Tv7VdzO88pfg==", "license": "Apache-2.0", "engines": { - "node": ">=20.16.0 || >=22.3.0" + "node": ">=20.19.0 || >=22.13.0 || >=24" }, "optionalDependencies": { - "@napi-rs/canvas": "^0.1.80" + "@napi-rs/canvas": "^0.1.96", + "node-readable-to-web-readable-stream": "^0.4.2" } }, "node_modules/picocolors": { @@ -11069,6 +10864,18 @@ } } }, + "node_modules/react-pdf/node_modules/pdfjs-dist": { + "version": "5.4.296", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.296.tgz", + "integrity": "sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=20.16.0 || >=22.3.0" + }, + "optionalDependencies": { + "@napi-rs/canvas": "^0.1.80" + } + }, "node_modules/recast": { "version": "0.23.11", "resolved": "https://registry.npmjs.org/recast/-/recast-0.23.11.tgz", diff --git a/frontend/package.json b/frontend/package.json index 764d9822..5e7a4191 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -15,7 +15,6 @@ "dependencies": { "@base-ui/react": "^1.4.1", "@tailwindcss/typography": "^0.5.19", - "@xyflow/react": "^12.11.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "date-fns": "^4.4.0", @@ -24,7 +23,7 @@ "lucide-react": "^1.8.0", "next": "16.2.4", "next-themes": "^0.4.6", - "pdfjs-dist": "5.4.296", + "pdfjs-dist": "^5.6.205", "react": "19.2.4", "react-dom": "19.2.4", "react-dropzone": "^15.0.0", diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index 300f0489..85d7192c 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -8,23 +8,12 @@ export default defineConfig({ timeout: 30_000, expect: { timeout: 5_000, - toHaveScreenshot: { - // Allow up to 2% of pixels to differ before failing - maxDiffPixelRatio: 0.02, - // Per-pixel colour difference threshold (0–1) - threshold: 0.2, - // Animations must be disabled before snapshotting - animations: "disabled", - }, }, fullyParallel: true, reporter: process.env.CI ? [["github"], ["list"]] : "list", use: { baseURL, trace: "on-first-retry", - // Consistent viewport for snapshot reproducibility - viewport: { width: 1280, height: 720 }, - colorScheme: "dark", }, webServer: { command: `npm run dev -- --hostname 127.0.0.1 --port ${port}`, @@ -38,7 +27,4 @@ export default defineConfig({ use: { ...devices["Desktop Chrome"] }, }, ], - // Store snapshots next to the spec files - snapshotPathTemplate: - "{testDir}/{testFilePath}-snapshots/{arg}-{projectName}{ext}", }); diff --git a/frontend/public/logo.jpg b/frontend/public/logo.jpg deleted file mode 100644 index b9e82ba8..00000000 --- a/frontend/public/logo.jpg +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bc0f9860b92fe5de3b660c3ff5dca4c17678b19186b768be83c37ebdcdb37370 -size 456354 diff --git a/frontend/src/app/auth/error/page.tsx b/frontend/src/app/auth/error/page.tsx deleted file mode 100644 index 5bdc7921..00000000 --- a/frontend/src/app/auth/error/page.tsx +++ /dev/null @@ -1,109 +0,0 @@ -"use client"; - -import { Suspense } from "react"; -import { useSearchParams, useRouter } from "next/navigation"; -import { Button } from "@/components/ui/button"; -import { - Card, - CardContent, - CardHeader, - CardTitle, - CardDescription, -} from "@/components/ui/card"; -import { AlertTriangle, RefreshCcw, Home } from "lucide-react"; - -const ERROR_MESSAGES: Record = { - csrf_mismatch: { - title: "Security Check Failed", - description: - "The OAuth state did not match. This could indicate a CSRF attack. Please try signing in again.", - }, - token_exchange_failed: { - title: "Token Exchange Failed", - description: - "We could not exchange your authorization code for an access token. Please try again.", - }, - userinfo_failed: { - title: "Profile Fetch Failed", - description: - "We could not retrieve your Hugging Face profile. Please check your account and try again.", - }, - email_required: { - title: "Email Required", - description: - "Your Hugging Face account did not provide an email address. Please ensure your email is public and try again.", - }, - oauth_not_configured: { - title: "OAuth Not Configured", - description: - "Hugging Face OAuth is not configured on this server. Please contact the administrator.", - }, - default: { - title: "Authentication Failed", - description: - "Something went wrong during sign-in with Hugging Face. Please try again.", - }, -}; - -function AuthErrorContent() { - const searchParams = useSearchParams(); - const router = useRouter(); - const errorCode = searchParams.get("error") ?? "default"; - const { title, description } = - ERROR_MESSAGES[errorCode] ?? ERROR_MESSAGES.default; - - return ( -
-
- - - -
-
- -
-
- {title} - - {description} - - - - - {errorCode !== "default" && ( -
- - error: {errorCode} - -
- )} - - - - - - -
- ); -} - -export default function AuthErrorPage() { - return ( - - - - ); -} \ No newline at end of file diff --git a/frontend/src/app/dashboard/page.tsx b/frontend/src/app/dashboard/page.tsx index 3d3392be..f5a25f0a 100644 --- a/frontend/src/app/dashboard/page.tsx +++ b/frontend/src/app/dashboard/page.tsx @@ -5,19 +5,11 @@ import dynamic from "next/dynamic"; import { useRouter } from "next/navigation"; import { useAuth } from "@/lib/auth"; import { toast } from "sonner"; -import { - api, - CONNECTION_ERROR_BANNER_MESSAGE, - CONNECTION_ERROR_MESSAGE, -} from "@/lib/api"; +import { api, CONNECTION_ERROR_BANNER_MESSAGE, CONNECTION_ERROR_MESSAGE } from "@/lib/api"; import Header from "@/components/layout/Header"; import DocumentSidebar from "@/components/document/DocumentSidebar"; import ChatSessionSidebar from "@/components/chat/ChatSessionSidebar"; import ChatPanel from "@/components/chat/ChatPanel"; -import CompareView from "@/components/document/CompareView"; -import DashboardDropOverlay from "@/components/document/DashboardDropOverlay"; -import { useDashboardDrop } from "@/hooks/useDashboardDrop"; - function PDFViewerSkeleton() { return (
import("@/components/document/PDFViewer"), { loading: () => , }); -// Lazy-load the graph panel — it pulls in @xyflow/react which is sizeable -const KnowledgeGraph = dynamic( - () => import("@/components/graph/KnowledgeGraph"), - { ssr: false }, -); - export interface DocInfo { chunk_size?: number; chunk_overlap?: number; @@ -67,11 +53,10 @@ export interface DocInfo { status: string; error_message: string | null; uploaded_at: string; - keywords?: string[]; } export default function DashboardPage() { - const { user, initialized } = useAuth(); + const { user, loading, initialized } = useAuth(); const router = useRouter(); const [documents, setDocuments] = useState([]); @@ -90,23 +75,18 @@ export default function DashboardPage() { } | null>(null); const [sidebarOpen, setSidebarOpen] = useState(true); const [viewerOpen, setViewerOpen] = useState(true); - const [graphOpen, setGraphOpen] = useState(false); const [connectionError, setConnectionError] = useState(""); const [documentsLoading, setDocumentsLoading] = useState(true); - const [compareOpen, setCompareOpen] = useState(false); const handleDocumentRenamed = useCallback((renamedDocument: DocInfo) => { setDocuments((current) => - current.map((document) => - document.id === renamedDocument.id ? renamedDocument : document, - ), - ); - setActiveDoc((current) => - current?.id === renamedDocument.id ? renamedDocument : current, + current.map((document) => (document.id === renamedDocument.id ? renamedDocument : document)) ); + setActiveDoc((current) => (current?.id === renamedDocument.id ? renamedDocument : current)); }, []); // Auth guard + useEffect(() => { if (initialized && !user) router.replace("/login"); }, [user, initialized, router]); @@ -115,30 +95,31 @@ export default function DashboardPage() { useEffect(() => { if (user) { const hasHfToken = !!(user.hf_token || localStorage.getItem("hf_token")); + if (!hasHfToken) { console.info( - "Hugging Face API token is not configured. Personal model access will fall back to the system default unless set in the user profile menu.", + "Hugging Face API token is not configured. Personal model access will fall back to the system default unless set in the user profile menu." ); } } }, [user]); + // Load documents const loadDocuments = useCallback(async () => { setDocumentsLoading(true); try { const data = await api.get<{ documents?: DocInfo[]; items?: DocInfo[] }>( - "/api/v1/documents/", + "/api/v1/documents/" ); setDocuments(data?.documents ?? data?.items ?? []); setConnectionError(""); } catch (err) { - const message = - err instanceof Error ? err.message : CONNECTION_ERROR_MESSAGE; + const message = err instanceof Error ? err.message : CONNECTION_ERROR_MESSAGE; setConnectionError( message === CONNECTION_ERROR_MESSAGE ? CONNECTION_ERROR_BANNER_MESSAGE - : `⚠️ ${message}`, + : `⚠️ ${message}` ); } finally { setDocumentsLoading(false); @@ -152,45 +133,19 @@ export default function DashboardPage() { })(); }, [user, loadDocuments]); - // ── Full-page drag-and-drop ────────────────────────────────────────────── - const handlePageDrop = useCallback( - async (files: File[]) => { - for (const file of files) { - const formData = new FormData(); - formData.append("file", file); - try { - await api.postForm("/api/v1/documents/upload", formData); - } catch (err) { - const message = err instanceof Error ? err.message : "Upload failed"; - console.error(`Upload failed for ${file.name}:`, message); - } - } - void loadDocuments(); - }, - [loadDocuments], - ); - - const { isDraggingOver, dropZoneProps } = useDashboardDrop({ - onDrop: handlePageDrop, - disabled: !user, - }); - // Ingest status change toast notification handler useEffect(() => { const prev = prevDocsRef.current; const nextPrevDocs: Record = {}; (documents || []).forEach((doc) => { nextPrevDocs[doc.id] = doc.status; + const oldStatus = prev[doc.id]; if (oldStatus && oldStatus !== doc.status) { if (doc.status === "ready") { - toast.success( - `🎉 Ingestion complete: '${doc.original_name}' is ready!`, - ); + toast.success(`🎉 Ingestion complete: '${doc.original_name}' is ready!`); } else if (doc.status === "failed") { - toast.error( - `❌ Ingestion failed for '${doc.original_name}': ${doc.error_message || "Unknown error"}`, - ); + toast.error(`❌ Ingestion failed for '${doc.original_name}': ${doc.error_message || "Unknown error"}`); } } }); @@ -200,20 +155,14 @@ export default function DashboardPage() { // Poll for processing status useEffect(() => { const hasPending = (documents || []).some( - (d) => d.status === "pending" || d.status === "processing", + (d) => d.status === "pending" || d.status === "processing" ); if (!hasPending) return; + const interval = setInterval(loadDocuments, 3000); return () => clearInterval(interval); }, [documents, loadDocuments]); - // Close graph panel when active document changes — derive from render - const prevDocIdRef = useRef(null); - if (activeDoc?.id !== prevDocIdRef.current) { - prevDocIdRef.current = activeDoc?.id ?? null; - if (graphOpen) setGraphOpen(false); - } - if (!initialized || !user) { return (
@@ -222,6 +171,7 @@ export default function DashboardPage() { ); } + // Shared sidebar content — used by both desktop panel and mobile sheet const sidebarContent = ( - +
setSidebarOpen(!sidebarOpen)} viewerOpen={viewerOpen} onToggleViewer={() => setViewerOpen(!viewerOpen)} - compareOpen={compareOpen} - onToggleCompare={() => setCompareOpen(!compareOpen)} mobileSheetContent={sidebarContent} /> @@ -262,58 +206,44 @@ export default function DashboardPage() { )}
- {/* ── Left: Document Sidebar — desktop only ───────────────────── */} + {/* ── Left: Document Sidebar — desktop only (md+) ─────────── */} {sidebarOpen && (
{sidebarContent}
)} - {/* ── Left-Center: Chat Sessions Sidebar ──────────────────────── */} + {/* ── Left-Center: Chat Sessions Sidebar ──── */} - {/* ── Center: Chat Panel ───────────────────────────────────────── */} + {/* ── Center: Chat Panel ──────────────────────────────────── */}
{ setPdfPage(target.page); - setPdfHighlightTarget({ - page: target.page, - rects: target.highlightRects, - }); + setPdfHighlightTarget({ page: target.page, rects: target.highlightRects }); if (!viewerOpen) setViewerOpen(true); }} />
- {/* ── Right: Compare View or Single PDF Viewer ────────────── */} - {compareOpen ? ( -
- setCompareOpen(false)} + {/* ── Right: PDF Viewer — hidden on mobile ────────────────── */} + {viewerOpen && activeDoc && activeDoc.original_name.endsWith(".pdf") && ( +
+ { + setPdfPage(page); + if (pdfHighlightTarget?.page !== page) { + setPdfHighlightTarget(null); + } + }} + totalPages={activeDoc.page_count} + highlightTarget={pdfHighlightTarget} />
- ) : ( - viewerOpen && - activeDoc && - activeDoc.original_name.endsWith(".pdf") && ( -
- { - setPdfPage(page); - if (pdfHighlightTarget?.page !== page) { - setPdfHighlightTarget(null); - } - }} - totalPages={activeDoc.page_count} - highlightTarget={pdfHighlightTarget} - /> -
- ) )}
diff --git a/frontend/src/app/favicon.ico b/frontend/src/app/favicon.ico index ccb5320d..718d6fea 100644 Binary files a/frontend/src/app/favicon.ico and b/frontend/src/app/favicon.ico differ diff --git a/frontend/src/app/globals.css b/frontend/src/app/globals.css index fbcacad4..5d65c047 100644 --- a/frontend/src/app/globals.css +++ b/frontend/src/app/globals.css @@ -372,19 +372,3 @@ .light .prose-chat .hljs-comment { color: oklch(0.5 0 0); } .light .prose-chat strong { color: oklch(0.2 0 0); } .light .prose-chat blockquote { color: oklch(0.4 0 0); } - -html, -body { - min-width: 375px; -} - -/* - * Enable iOS safe-area insets so fixed elements (FAB, modals) clear - * the home indicator bar on iPhone SE and similar devices. - * Requires viewport-fit=cover in the tag. - */ -@supports (padding: env(safe-area-inset-bottom)) { - :root { - --safe-area-bottom: env(safe-area-inset-bottom); - } -} \ No newline at end of file diff --git a/frontend/src/app/layout.tsx b/frontend/src/app/layout.tsx index cb87453f..456ce685 100644 --- a/frontend/src/app/layout.tsx +++ b/frontend/src/app/layout.tsx @@ -5,6 +5,7 @@ import { AuthProvider } from "@/lib/auth"; import { TooltipProvider } from "@/components/ui/tooltip"; import I18nProvider from "@/components/providers/I18nProvider"; import { ThemeProvider } from "@/components/layout/ThemeProvider"; +import { Toaster } from "sonner"; const inter = Inter({ variable: "--font-sans", diff --git a/frontend/src/app/login/page.tsx b/frontend/src/app/login/page.tsx index 299922ba..5801dc89 100644 --- a/frontend/src/app/login/page.tsx +++ b/frontend/src/app/login/page.tsx @@ -7,7 +7,7 @@ import { useTranslation } from "react-i18next"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/components/ui/card"; -import { Brain, Eye, EyeOff, Loader2 } from "lucide-react"; +import { Brain, Eye, EyeOff } from "lucide-react"; import Link from "next/link"; import GoogleSignInButton from "@/components/auth/GoogleSignInButton"; import HuggingFaceSignInButton from "@/components/auth/HuggingFaceSignInButton"; @@ -16,7 +16,6 @@ export default function LoginPage() { const { login, user, initialized } = useAuth(); const { t } = useTranslation(); const router = useRouter(); - const [email, setEmail] = useState(""); const [password, setPassword] = useState(""); const [showPw, setShowPw] = useState(false); @@ -45,15 +44,16 @@ export default function LoginPage() { } catch (err: unknown) { const message = err instanceof Error ? err.message : t("login.fallbackError"); setError(message); - setLoading(false); // Keeps loading indicator active only until redirect happens + } finally { + setLoading(false); } }; return ( -
+
{/* Background glow */}
- +
@@ -64,12 +64,14 @@ export default function LoginPage() { {t("login.title")} {t("login.description")} - + - {/* Social Sign-In Buttons */}
- +
@@ -85,18 +87,13 @@ export default function LoginPage() {
{error && ( -
+
{error}
)}
- + setEmail(e.target.value)} required className="h-11" - autoComplete="email" />
- +
setPassword(e.target.value)} required className="h-11 pr-10" - autoComplete="current-password" />
- + )} - {/* ── Content ─────────────────────────────────────────────────────── */} -
+ {/* ── Hero ────────────────────────────────────── */} +
+ {/* Glow effect */} +
- {/* Badge */} -
- +
+
+ Enterprise Agentic RAG System
- {/* Headline */} -

+

Chat with your{" "} - + documents - -
- - intelligently. - + {" "} + intelligently

- {/* Divider glow line */} -
- - {/* Sub-copy */} -

+

Upload financial reports, legal contracts, or research papers and get - accurate, cited insights powered - by advanced AI retrieval. + accurate, cited insights powered by advanced AI retrieval.

- {/* CTAs */} -
+
- - -
- - {/* Social proof strip */} -

- No credit card required  ·  Open-source  ·  Self-hostable -

- {/* ── Feature cards ────────────────────────────────────────────────── */} -
- {FEATURES.map(({ icon: Icon, title, desc }, i) => ( + {/* ── Features Grid ────────────────────────── */} +
+ {[ + { + icon: FileText, + title: "Multi-Format Upload", + desc: "PDF, DOCX, TXT, and Markdown with smart chunking", + }, + { + icon: Search, + title: "Semantic Search", + desc: "Two-stage retrieval with cross-encoder reranking", + }, + { + icon: MessageSquare, + title: "Streaming Chat", + desc: "Real-time AI responses with source citations", + }, + { + icon: Zap, + title: "Instant Insights", + desc: "Extract key facts, summaries, and data points", + }, + { + icon: Shield, + title: "Data Isolation", + desc: "Per-user vector collections for complete privacy", + }, + { + icon: Brain, + title: "Open-Source LLMs", + desc: "Powered by Mistral and HuggingFace ecosystem", + }, + ].map((f, i) => (
- {/* Icon with subtle glow on hover */} -
- -
- -

{title}

-

{desc}

+ +

{f.title}

+

{f.desc}

))}
- - - {/* ── Footer ────────────────────────────────────────────────────────── */} -
-
- {/* Brand info */} -
-
- Logo - - Document AI Analyst - -
-

- Enterprise Retrieval-Augmented Generation (RAG) platform. Analyze complex PDF documents and chat with intelligence. -

-
- - {/* Links column 1 */} -
-

Product

- -
+
- {/* Links column 2 */} -
-

Legal & Privacy

-
    -
  • - Privacy Policy -
    • -
  • - Terms of Service -
    • -
+ {/* ── Footer ──────────────────────────────────── */} +
+
+ Built with FastAPI • LangChain • ChromaDB • HuggingFace • Next.js +
+ + Privacy Policy + + + Terms of Service +
- -
-

- © {new Date().getFullYear()} Document AI Analyst. All rights reserved. -

-

- Built with - {["FastAPI", "LangChain", "ChromaDB", "HuggingFace", "Next.js"].map((tech, i, arr) => ( - - {tech} - {i < arr.length - 1 && ·} - - ))} -

-
- {/* Hall of Fame modal */} + {/* Hall of Fame Modal */} {hallOfFameOpen && ( setHallOfFameOpen(false)} /> )} - {/* Floating open-source badge */} + {/* Open Source floating badge */} setHallOfFameOpen(true)} /> - - {/* Maintenance Pop-up Banner */} - {maintenanceVisible && ( -
-
-
- -

System Maintenance

-
- -
-

- We are currently running database updates. Some features may be temporarily offline, but the app will be fully live soon! -

-
- )}
); -} +} \ No newline at end of file diff --git a/frontend/src/app/register/page.tsx b/frontend/src/app/register/page.tsx index 3872b4a3..f74e7dd1 100644 --- a/frontend/src/app/register/page.tsx +++ b/frontend/src/app/register/page.tsx @@ -6,17 +6,12 @@ import { useAuth } from "@/lib/auth"; import { useTranslation } from "react-i18next"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; -import { - Card, - CardContent, - CardHeader, - CardTitle, - CardDescription, -} from "@/components/ui/card"; +import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/components/ui/card"; import { Brain } from "lucide-react"; import Link from "next/link"; import GoogleSignInButton from "@/components/auth/GoogleSignInButton"; import { PasswordField } from "@/components/auth/PasswordField"; +import { isPasswordValid } from "@/lib/password-validation"; import HuggingFaceSignInButton from "@/components/auth/HuggingFaceSignInButton"; export default function RegisterPage() { @@ -32,6 +27,8 @@ export default function RegisterPage() { const [verificationUrl, setVerificationUrl] = useState(""); const [loading, setLoading] = useState(false); + const passwordValid = isPasswordValid(password); + const canSubmit = username.trim().length >= 3 && email.trim().length > 0 && passwordValid && !loading; // Redirect if already logged in useEffect(() => { if (initialized && user) { @@ -56,8 +53,7 @@ export default function RegisterPage() { setSuccess(result.message); setVerificationUrl(result.verification_url ?? ""); } catch (err: unknown) { - const message = - err instanceof Error ? err.message : t("register.fallbackError"); + const message = err instanceof Error ? err.message : t("register.fallbackError"); setError(message); } finally { setLoading(false); @@ -75,9 +71,7 @@ export default function RegisterPage() {
- - {t("register.title")} - + {t("register.title")} {t("register.description")} @@ -106,10 +100,7 @@ export default function RegisterPage() { {t("register.verifyMessage", { email: registeredEmail })}

{verificationUrl && ( - + {t("register.openVerification")} )} @@ -157,11 +148,7 @@ export default function RegisterPage() { disabled={Boolean(success)} /> - - ); - })} -
- -

Account

diff --git a/frontend/src/app/terms/page.tsx b/frontend/src/app/terms/page.tsx index aed83839..1aac1c0d 100644 --- a/frontend/src/app/terms/page.tsx +++ b/frontend/src/app/terms/page.tsx @@ -1,3 +1,4 @@ + import Link from "next/link"; import { ArrowLeft, diff --git a/frontend/src/components/auth/HuggingFaceTokenModal.tsx b/frontend/src/components/auth/HuggingFaceTokenModal.tsx index d53dbfb5..b3fb1321 100644 --- a/frontend/src/components/auth/HuggingFaceTokenModal.tsx +++ b/frontend/src/components/auth/HuggingFaceTokenModal.tsx @@ -13,9 +13,10 @@ import { DialogTrigger, } from "@/components/ui/dialog"; import { useAuthStore } from "@/store/auth-store"; -import { Eye, EyeOff, AlertCircle, CheckCircle2, Loader2, ExternalLink, Key, Trash2 } from "lucide-react"; +import { Eye, EyeOff, AlertCircle, CheckCircle2, Loader2, ExternalLink, Key } from "lucide-react"; interface HuggingFaceTokenModalProps { + /** Optional — if provided, allows a button-triggered dialog pattern */ children?: ReactNode; } @@ -29,7 +30,6 @@ export default function HuggingFaceTokenModal({ children }: HuggingFaceTokenModa const [open, setOpen] = useState(false); const [inputToken, setInputToken] = useState(existingToken); const [saving, setSaving] = useState(false); - const [removing, setRemoving] = useState(false); const [error, setError] = useState(null); const [success, setSuccess] = useState(false); const [showToken, setShowToken] = useState(false); @@ -37,6 +37,7 @@ export default function HuggingFaceTokenModal({ children }: HuggingFaceTokenModa const mountedRef = useRef(true); const timeoutRef = useRef | null>(null); + // Cleanup auto-close timeout and unmount guard on unmount useEffect(() => { return () => { mountedRef.current = false; @@ -58,44 +59,34 @@ export default function HuggingFaceTokenModal({ children }: HuggingFaceTokenModa clearAutoCloseTimeout(); setOpen(newOpen); if (newOpen) { + // Reset to current store value when opening (picks up changes from background saves) const currentToken = useAuthStore.getState().user?.hf_token ?? ""; setInputToken(currentToken); setSaving(false); - setRemoving(false); setError(null); setSuccess(false); setShowToken(false); } }; - const validateToken = (token: string): string | null => { - if (!token) return "Please enter a valid token"; - if (!token.startsWith("hf_")) return "Token must start with 'hf_'"; - if (token.length < 20) return "Token is too short — please check and try again"; - return null; - }; - - const getTokenPreview = (token: string): string => { - if (token.length <= 8) return token; - return `${token.slice(0, 7)}****${token.slice(-4)}`; - }; - const handleSave = async () => { if (saving) return; const token = inputToken.trim(); - const validationError = validateToken(token); - if (validationError) { - setError(validationError); + if (!token) { + setError("Please enter a valid token"); return; } + setSaving(true); setError(null); setSuccess(false); + try { await setHfToken(token); if (!mountedRef.current) return; setSaving(false); setSuccess(true); + // Auto-close after 1.5s timeoutRef.current = setTimeout(() => setOpen(false), 1500); } catch (err) { if (!mountedRef.current) return; @@ -104,26 +95,7 @@ export default function HuggingFaceTokenModal({ children }: HuggingFaceTokenModa } }; - const handleRemove = async () => { - if (removing) return; - setRemoving(true); - setError(null); - setSuccess(false); - try { - await setHfToken(""); - if (!mountedRef.current) return; - setRemoving(false); - setInputToken(""); - setSuccess(true); - timeoutRef.current = setTimeout(() => setOpen(false), 1500); - } catch (err) { - if (!mountedRef.current) return; - setRemoving(false); - setError(err instanceof Error ? err.message : "Failed to remove token"); - } - }; - - const isSaveDisabled = inputToken.trim() === "" || saving || removing; + const isSaveDisabled = inputToken.trim() === "" || saving; return ( @@ -150,127 +122,91 @@ export default function HuggingFaceTokenModal({ children }: HuggingFaceTokenModa { e.preventDefault(); if (!isSaveDisabled) handleSave(); }}> -
- -
- - {hasExistingToken && ( - - - Token configured - - )} -
- +
+ {/* Token label with configured indicator */} +
+ {hasExistingToken && ( -
- - Current: {getTokenPreview(existingToken)} -
+ + + Token configured + )} - -
- { - setInputToken(e.target.value); - if (error) setError(null); - if (success) setSuccess(false); - }} - placeholder="hf_..." - className="pr-10 font-mono" - disabled={saving || removing} - autoFocus - aria-label="HuggingFace API Token" - /> - -
- -

- Token must start with{" "} - hf_ -

- - - - - Get your API token from HuggingFace Settings - -
- {error && ( -
- - {error} -
- )} - - {success && ( -
- - {inputToken ? "Token saved successfully" : "Token removed successfully"} -
- )} - - - - {hasExistingToken && ( + {/* Input wrapper with visibility toggle */} +
+ { + setInputToken(e.target.value); + if (error) setError(null); + if (success) setSuccess(false); + }} + placeholder="hf_..." + className="pr-10 font-mono" + disabled={saving} + autoFocus + aria-label="HuggingFace API Token" + /> - )} +
-
+ + {/* Error banner */} + {error && ( +
+ + {error} +
+ )} + + {/* Success banner */} + {success && ( +
+ + Token saved successfully +
+ )} + + + {/* Footer */} + + -
); -} \ No newline at end of file +} diff --git a/frontend/src/components/chat/ChatPanel.tsx b/frontend/src/components/chat/ChatPanel.tsx index a4249c0f..25aceacc 100644 --- a/frontend/src/components/chat/ChatPanel.tsx +++ b/frontend/src/components/chat/ChatPanel.tsx @@ -5,33 +5,15 @@ import { useState, useRef, useEffect } from "react"; import { useTranslation } from "react-i18next"; import type { DocInfo } from "@/app/dashboard/page"; import { api, API_BASE } from "@/lib/api"; -import { - useChatStore, - type ChatMsg, - type SourceBoundingBox, - type SourceChunk, -} from "@/store/chat-store"; -import { buttonVariants } from "@/components/ui/button"; +import { useChatStore, type ChatMsg, type SourceBoundingBox, type SourceChunk } from "@/store/chat-store"; +import { Button } from "@/components/ui/button"; import { Skeleton } from "@/components/ui/skeleton"; import { Textarea } from "@/components/ui/textarea"; -import { - Tooltip, - TooltipContent, - TooltipTrigger, -} from "@/components/ui/tooltip"; import MessageBubble from "./MessageBubble"; import SourceCard from "./SourceCard"; -import { - Send, - Loader2, - Trash2, - MessageSquare, - Download, - Mic, - MicOff, - ChevronDown, -} from "lucide-react"; +import { Send, Loader2, Trash2, MessageSquare, Download, Mic, MicOff, HelpCircle } from "lucide-react"; import { cn } from "@/lib/utils"; + interface ISpeechRecognitionEvent { resultIndex: number; results: { @@ -89,25 +71,23 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) { const setStreaming = useChatStore((state) => state.setStreaming); const setIsTyping = useChatStore((state) => state.setIsTyping); const resetChat = useChatStore((state) => state.resetChat); - const fetchSessionHistory = useChatStore( - (state) => state.fetchSessionHistory, - ); - + const fetchSessionHistory = useChatStore((state) => state.fetchSessionHistory); + const [showExportMenu, setShowExportMenu] = useState(false); const MAX_CHARACTERS = 2000; const [isRecording, setIsRecording] = useState(false); const [speechError, setSpeechError] = useState(null); - - const [showScrollButton, setShowScrollButton] = useState(false); + + // New State for Keyboard Shortcuts Help Modal + const [showHelpModal, setShowHelpModal] = useState(false); const recognitionRef = useRef(null); const initialInputRef = useRef(""); const textareaRef = useRef(null); const bottomRef = useRef(null); - const containerRef = useRef(null); const prevDocId = useRef(null); const exportMenuRef = useRef(null); - const abortControllerRef = useRef(null); + const showEmptyState = messages.length === 0 && !isTyping && !historyLoading; useEffect(() => { @@ -116,7 +96,7 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) { textarea.style.height = "auto"; const computedMaxHeight = Number.parseFloat( - window.getComputedStyle(textarea).maxHeight, + window.getComputedStyle(textarea).maxHeight ); const maxHeight = Number.isFinite(computedMaxHeight) ? computedMaxHeight @@ -129,31 +109,9 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) { // Auto-scroll to bottom whenever messages change useEffect(() => { - if (containerRef.current) { - const { scrollHeight, scrollTop, clientHeight } = containerRef.current; - if (scrollHeight - scrollTop - clientHeight < 150) { - bottomRef.current?.scrollIntoView({ behavior: "smooth" }); - } - } else { - bottomRef.current?.scrollIntoView({ behavior: "smooth" }); - } + bottomRef.current?.scrollIntoView({ behavior: "smooth" }); }, [messages]); - const handleScroll = () => { - if (!containerRef.current) return; - const { scrollTop, scrollHeight, clientHeight } = containerRef.current; - setShowScrollButton(scrollTop < scrollHeight - clientHeight - 100); - }; - - const scrollToBottom = () => { - if (containerRef.current) { - containerRef.current.scrollTo({ - top: containerRef.current.scrollHeight, - behavior: "smooth", - }); - } - }; - useEffect(() => { return () => { resetChat(); @@ -181,14 +139,9 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) { let cancelled = false; api - .get<{ - messages: Array<{ - id: string; - role: string; - content: string; - sources?: SourceChunk[]; - }>; - }>(`/api/v1/chat/history/${documentId}`) + .get<{ messages: Array<{ id: string; role: string; content: string; sources?: SourceChunk[] }> }>( + `/api/v1/chat/history/${documentId}` + ) .then((data) => { if (cancelled || prevDocId.current !== documentId) return; @@ -198,7 +151,7 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) { role: m.role as "user" | "assistant", content: m.content, sources: m.sources || [], - })), + })) ); }) .catch(() => { @@ -211,20 +164,12 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) { }; }, [activeSessionId, activeDoc, fetchSessionHistory, setMessages]); - const handleStop = () => { - abortControllerRef.current?.abort(); - setStreaming(false); - setIsTyping(false); - }; - const handleSend = async () => { if (!input.trim() || streaming) return; const question = input.trim(); setInput(""); - const abortController = new AbortController(); - abortControllerRef.current = abortController; // Add user message const userMsg: ChatMsg = { id: `user-${Date.now()}`, @@ -241,262 +186,86 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) { setIsTyping(true); try { - // Try WebSocket first for real-time agentic thought streaming - const token = - typeof window !== "undefined" ? localStorage.getItem("token") : null; - const base = API_BASE || window.location.origin; - const wsScheme = base.startsWith("https") - ? "wss" - : base.startsWith("http") - ? "ws" - : "wss"; - const host = base.replace(/^https?:/, ""); - const wsUrl = `${wsScheme}:${host}/api/v1/chat/ws${token ? `?token=${encodeURIComponent(token)}` : ""}`; - - const ws = new WebSocket(wsUrl); - - let onAbort: (() => void) | null = null; - - const wsDone = new Promise((resolve, reject) => { - onAbort = () => { - try { - ws.close(); - } catch { - // ignore - } - reject(new DOMException("The user aborted a request.", "AbortError")); - }; - abortController.signal.addEventListener("abort", onAbort); - - ws.onopen = () => { - // Send initial payload - ws.send( - JSON.stringify({ - question, - document_id: activeDoc?.id || null, - session_id: activeSessionId, - }), - ); - }; - - // If WS doesn't open within 800ms, treat as failure and fallback - const connectTimeout = setTimeout(() => { - try { - ws.close(); - } catch { - // ignore - } - reject(new Error("WebSocket connection timeout")); - }, 800); - - ws.onmessage = (ev) => { - clearTimeout(connectTimeout); - try { - const event = JSON.parse(ev.data); - if (event.type === "token") { - if (!assistantCreated) { - assistantCreated = true; - setIsTyping(false); - - const assistantMsg: ChatMsg = { - id: assistantId, - role: "assistant", - content: event.data as string, - sources: [], - isStreaming: true, - }; - - setMessages((prev) => [...prev, assistantMsg]); - } else { - setMessages((prev) => - prev.map((m) => - m.id === assistantId - ? { ...m, content: m.content + (event.data as string) } - : m, - ), - ); - } - } else if (event.type === "sources") { - setMessages((prev) => - prev.map((m) => - m.id === assistantId - ? { ...m, sources: event.data as SourceChunk[] } - : m, - ), - ); - } else if (event.type === "thought") { - // Append thoughts as a temporary assistant note (optional UI handling) - // For simplicity, add to assistant message content in brackets - setMessages((prev) => - prev.map((m) => - m.id === assistantId - ? { ...m, content: m.content + `\n[thought] ${event.data}` } - : m, - ), - ); - } else if (event.type === "error") { - setIsTyping(false); - setMessages((prev) => - prev.map((m) => - m.id === assistantId - ? { - ...m, - content: `Error: ${event.data}`, - isStreaming: false, - } - : m, - ), - ); - ws.close(); - reject(new Error(String(event.data))); - } else if (event.type === "done") { - setMessages((prev) => - prev.map((m) => - m.id === assistantId - ? { - ...m, - isStreaming: false, - response_time_ms: event.response_time_ms, - } - : m, - ), - ); - ws.close(); - resolve(); - } - } catch { - // ignore malformed messages - } - }; - - ws.onerror = () => { - clearTimeout(connectTimeout); - reject(new Error("WebSocket error")); - }; - - ws.onclose = () => { - resolve(); - }; + const stream = api.streamPost("/api/v1/chat/ask/stream", { + question, + document_id: activeDoc?.id || null, + session_id: activeSessionId, }); - try { - await wsDone; - } finally { - if (onAbort) { - abortController.signal.removeEventListener("abort", onAbort); - } - } - } catch (err) { - if ( - err instanceof Error && - (err.name === "AbortError" || - err.message === "The user aborted a request.") - ) { - return; - } - // Fallback to existing SSE stream if WebSocket fails - try { - const stream = api.streamPost( - "/api/v1/chat/ask/stream", - { - question, - document_id: activeDoc?.id || null, - session_id: activeSessionId, - }, - abortController.signal, - ); - - for await (const event of stream) { - if (event.type === "token") { - if (!assistantCreated) { - assistantCreated = true; - setIsTyping(false); + for await (const event of stream) { + if (event.type === "token") { + // Create assistant message only when first token arrives + if (!assistantCreated) { + assistantCreated = true; + setIsTyping(false); - const assistantMsg: ChatMsg = { - id: assistantId, - role: "assistant", - content: event.data as string, - sources: [], - isStreaming: true, - }; + const assistantMsg: ChatMsg = { + id: assistantId, + role: "assistant", + content: event.data as string, + sources: [], + isStreaming: true, + }; - setMessages((prev) => [...prev, assistantMsg]); - } else { - setMessages((prev) => - prev.map((m) => - m.id === assistantId - ? { ...m, content: m.content + (event.data as string) } - : m, - ), - ); - } - } else if (event.type === "sources") { + setMessages((prev) => [...prev, assistantMsg]); + } else { setMessages((prev) => prev.map((m) => m.id === assistantId - ? { ...m, sources: event.data as SourceChunk[] } - : m, - ), - ); - } else if (event.type === "error") { - setIsTyping(false); - setMessages((prev) => - prev.map((m) => - m.id === assistantId - ? { - ...m, - content: `Error: ${event.data}`, - isStreaming: false, - } - : m, - ), - ); - } else if (event.type === "done") { - setMessages((prev) => - prev.map((m) => - m.id === assistantId - ? { - ...m, - isStreaming: false, - response_time_ms: ( - event as { type: string; response_time_ms?: number } - ).response_time_ms, - } - : m, - ), + ? { ...m, content: m.content + (event.data as string) } + : m + ) ); } - } - } catch (err2) { - setIsTyping(false); - if ( - err2 instanceof Error && - (err2.name === "AbortError" || - err2.message === "The user aborted a request.") - ) { + } else if (event.type === "sources") { + setMessages((prev) => + prev.map((m) => + m.id === assistantId + ? { ...m, sources: event.data as SourceChunk[] } + : m + ) + ); + } else if (event.type === "error") { + setIsTyping(false); + setMessages((prev) => + prev.map((m) => + m.id === assistantId + ? { ...m, content: `Error: ${event.data}`, isStreaming: false } + : m + ) + ); + } else if (event.type === "done") { setMessages((prev) => prev.map((m) => - m.id === assistantId ? { ...m, isStreaming: false } : m, - ), + m.id === assistantId ? { ...m, isStreaming: false } : m + ) ); - return; } - setMessages((prev) => - prev.map((m) => - m.id === assistantId - ? { - ...m, - content: t("chat.fallbackError", { - message: - err2 instanceof Error ? err2.message : "Unknown error", - }), - isStreaming: false, - } - : m, - ), - ); + } + } catch (err) { + const message = err instanceof Error ? err.message : "Unknown error"; + setIsTyping(false); + setMessages((prev) => + prev.map((m) => + m.id === assistantId + ? { + ...m, + content: t("chat.fallbackError", { + message, + }), + isStreaming: false, + } + : m + ) + ); + + if (message.toLowerCase().includes("connect") || message.toLowerCase().includes("network")) { + toast.error("Network error. Please check your connection."); + } else { + toast.error(`Upload failed: ${message}`); } } finally { + setStreaming(false); setIsTyping(false); } @@ -531,10 +300,7 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) { useEffect(() => { if (!showExportMenu) return; const handleClickOutside = (e: MouseEvent) => { - if ( - exportMenuRef.current && - !exportMenuRef.current.contains(e.target as Node) - ) { + if (exportMenuRef.current && !exportMenuRef.current.contains(e.target as Node)) { setShowExportMenu(false); } }; @@ -559,11 +325,7 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) { : null; if (!SpeechRecognitionAPI) { - setSpeechError( - t("chat.speechNotSupported", { - defaultValue: "Speech recognition is not supported in this browser.", - }), - ); + setSpeechError(t("chat.speechNotSupported", { defaultValue: "Speech recognition is not supported in this browser." })); return; } @@ -571,7 +333,7 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) { const recognition = new SpeechRecognitionAPI(); recognition.continuous = true; recognition.interimResults = true; - + const currentLang = i18n.language || "en"; const langMap: Record = { en: "en-US", @@ -593,7 +355,7 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) { setInput( initialInputRef.current + (initialInputRef.current ? " " : "") + - sessionTranscript.trim(), + sessionTranscript.trim() ); }; @@ -601,13 +363,10 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) { const errorCode = event.error; if (errorCode === "aborted") return; // ignore manual aborts - let msg = t("chat.speechError", { - defaultValue: `Speech recognition error: ${errorCode}`, - }); + let msg = t("chat.speechError", { defaultValue: `Speech recognition error: ${errorCode}` }); if (errorCode === "not-allowed") { msg = t("chat.micPermissionDenied", { - defaultValue: - "Microphone access denied. Please enable permissions in settings.", + defaultValue: "Microphone access denied. Please enable permissions in settings.", }); } else if (errorCode === "no-speech") { msg = t("chat.noSpeechDetected", { @@ -633,11 +392,7 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) { recognitionRef.current = recognition; recognition.start(); } catch (err) { - setSpeechError( - err instanceof Error - ? err.message - : "Failed to start speech recognition.", - ); + setSpeechError(err instanceof Error ? err.message : "Failed to start speech recognition."); setIsRecording(false); } }; @@ -670,16 +425,11 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) { }; // ── NEW KEYBOARD SHORTCUTS ENGINE EFFECT ────────────────────────── - // ── KEYBOARD SHORTCUTS ENGINE EFFECT ────────────────────────── useEffect(() => { const handleGlobalKeyDown = (e: KeyboardEvent) => { const isCmdOrCtrl = e.metaKey || e.ctrlKey; - const isMobile = /Mobi|Android/i.test(navigator.userAgent); - - // No-op on mobile devices - if (isMobile) return; - // Shortcut 1: Ctrl/Cmd + Enter → Send Message (when textarea focused) + // Shortcut 1: Ctrl/Cmd + Enter -> Send Message (When textarea has focus) if (isCmdOrCtrl && e.key === "Enter") { if (document.activeElement === textareaRef.current) { e.preventDefault(); @@ -687,87 +437,42 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) { } } - // Shortcut 2: Escape → Abort SSE stream OR clear input OR close modal + // Shortcut 2: Escape -> Clear Input / Close Modal if (e.key === "Escape") { - if (streaming) { - e.preventDefault(); - handleStop(); - toast.info("Response cancelled"); - } else if (document.activeElement === textareaRef.current) { + if (document.activeElement === textareaRef.current) { e.preventDefault(); - setInput(""); - } else if (showExportMenu) { - setShowExportMenu(false); + setInput(""); // Clear textarea state + } else if (showHelpModal) { + setShowHelpModal(false); // Close shortcuts modal if open } } - // Shortcut 3: Ctrl/Cmd + K → Focus chat input from anywhere + // Shortcut 3: Ctrl/Cmd + K -> Focus chat input from anywhere if (isCmdOrCtrl && (e.key === "k" || e.key === "K")) { e.preventDefault(); textareaRef.current?.focus(); } - - // Shortcut 5: Ctrl/Cmd + Shift + C → Clear chat history - if (isCmdOrCtrl && e.shiftKey && (e.key === "c" || e.key === "C")) { - e.preventDefault(); - handleClear(); - } - - // Shortcut 6: Ctrl/Cmd + Shift + E → Toggle export menu - if (isCmdOrCtrl && e.shiftKey && (e.key === "e" || e.key === "E")) { - e.preventDefault(); - if (messages.length > 0) { - setShowExportMenu((prev) => !prev); - } - } - - // Shortcut 7: Ctrl/Cmd + Shift + M → Toggle mic recording - if (isCmdOrCtrl && e.shiftKey && (e.key === "m" || e.key === "M")) { - e.preventDefault(); - if (!streaming) { - toggleRecording(); - } - } }; window.addEventListener("keydown", handleGlobalKeyDown); return () => { window.removeEventListener("keydown", handleGlobalKeyDown); }; - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [input, streaming, showExportMenu, messages]); // Dependencies updated to capture fresh state data + }, [input, streaming, showHelpModal]); // Dependencies updated to capture fresh state data return (
{/* ── Chat Messages ──────────────────────────── */} -
+
{historyLoading ? ( -
+
{Array.from({ length: 4 }).map((_, index) => (
- {index % 2 !== 0 && ( - - )} -
+ {index % 2 !== 0 && } +
@@ -781,9 +486,7 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) {

- {activeDoc - ? t("chat.askAboutDocument") - : t("chat.selectDocument")} + {activeDoc ? t("chat.askAboutDocument") : t("chat.selectDocument")}

{activeDoc @@ -798,10 +501,7 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) { {msg.role === "assistant" && msg.sources.length > 0 && (

- +
)}
@@ -818,23 +518,8 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) {
- {/* Scroll to bottom button */} - - {/* ── Input Area ─────────────────────────────── */} -
+
{/* Status / Error Message Area */} {(isRecording || speechError) && ( @@ -847,15 +532,11 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) { - {t("chat.listening", { - defaultValue: "Listening... Speak now.", - })} + {t("chat.listening", { defaultValue: "Listening... Speak now." })} ) : ( - - {speechError} - + {speechError} )}
@@ -888,9 +567,7 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) { onKeyDown={handleKeyDown} placeholder={ activeDoc - ? t("chat.askPlaceholder", { - name: activeDoc.original_name, - }) + ? t("chat.askPlaceholder", { name: activeDoc.original_name }) : t("chat.selectPlaceholder") } disabled={streaming} @@ -899,102 +576,91 @@ export default function ChatPanel({ activeDoc, onCitationClick }: Props) { aria-label="Chat message" aria-describedby="chat-input-hint" /> - + {/* Mic Button */} - - - {isRecording ? ( - - ) : ( - - )} - - - {isRecording - ? t("chat.stopRecording", { - defaultValue: "Stop recording", - }) - : t("chat.startRecording", { - defaultValue: "Start recording", - })} - - -
+ + {/* NEW Keyboard Shortcuts Info Button */} + +
+
- - - {streaming ? ( - - ) : ( - - )} - - - {streaming ? "Stop generating" : "Send message"} - - + {messages.length > 0 && ( <> {/* Export dropdown */}
- - setShowExportMenu((v) => !v)} - className={cn( - buttonVariants({ variant: "ghost", size: "icon" }), - "h-10 w-10 sm:h-[44px] sm:w-[44px] text-muted-foreground hover:text-primary", - )} - aria-label={t("chat.exportTitle")} - aria-expanded={showExportMenu} - aria-controls="chat-export-menu" - aria-haspopup="menu" - > - - - {t("chat.exportTitle")} - + {showExportMenu && (

- Press Enter to send. Press Shift and Enter for a new line. Press - Ctrl+Enter to send from anywhere. Press Escape to cancel streaming. + Press Enter to send. Press Shift and Enter for a new line.

+ + {/* ── NEW KEYBOARD SHORTCUTS HELP MODAL OVERLAY ───────────────── */} + {showHelpModal && ( +
setShowHelpModal(false)} + > +
e.stopPropagation()} // Stop overlay closing when clicking inside + > + + +

+ ⌨️ Keyboard Shortcuts +

+

Enhance your typing productivity

+
+ +
    +
  • + Send Message +
    + Ctrl + + + Enter +
    +
    • +
  • + Clear Chat Input +
    + Esc +
    +
    • +
  • + Focus Chat Input +
    + Ctrl + + + K +
    +
    • +
+
+
+ )}
); -} +} \ No newline at end of file diff --git a/frontend/src/components/chat/ChatSessionSidebar.tsx b/frontend/src/components/chat/ChatSessionSidebar.tsx index e2bd96f8..d2080f68 100644 --- a/frontend/src/components/chat/ChatSessionSidebar.tsx +++ b/frontend/src/components/chat/ChatSessionSidebar.tsx @@ -235,16 +235,9 @@ export default function ChatSessionSidebar() {
- {/* - FAB — mobile only (md:hidden). - z-20: sits below the Header (z-50) and its backdrop (z-40) so - it is not tappable when the document navigation sheet is open. - bottom uses safe-area-inset-bottom so the button clears the iOS - home indicator on iPhone SE and other small-screen devices. - */} + )} - {/* Copy button */} - - - {copied ? ( - - ) : ( - - )} - - - {copied ? "Copied" : "Copy response"} - - + {/* Speech button */} + {!message.isStreaming && ( + + )} - {/* Share button */} - {!message.isStreaming && ( - - - {shared ? ( - - ) : shareFailed ? ( - - ) : ( - - )} - - - {shared - ? "Link copied" - : shareFailed - ? "Share failed" - : "Share response"} - - + {/* Copy button */} +
- + {copied && ( -
Copied!
)} + + {/* Play / Pause button */} + )} -
+
{message.content ? ( )}
+ {!message.isStreaming && !isUser && ( +
+ Was this helpful? + + +
+ )} )} - -
-
- {formatDistanceToNow(new Date(Number(message.id.split("-")[1])), { - addSuffix: true, - })} -
- {!isUser && message.response_time_ms && ( - - ⚡ {(message.response_time_ms / 1000).toFixed(1)}s - + {formatDistanceToNow( + new Date(Number(message.id.split("-")[1])), + { addSuffix: true } )}
- {isUser && ( + {isUser && (
)}
); -} +} \ No newline at end of file diff --git a/frontend/src/components/chat/SourceCard.tsx b/frontend/src/components/chat/SourceCard.tsx index 34f458d9..0840a185 100644 --- a/frontend/src/components/chat/SourceCard.tsx +++ b/frontend/src/components/chat/SourceCard.tsx @@ -64,14 +64,6 @@ const getConfidenceBadgeMeta = (value?: number): ConfidenceBadgeMeta => { const getPrimarySourceMetric = (source: SourceChunk) => source.confidence ?? source.score; -// Returns true only when the source has a real, usable page number. -// Some document types (e.g. .txt, .md) have no page concept, and the -// backend may send page as undefined/null/NaN in those cases. -const hasValidPage = (source: SourceChunk) => - typeof source.page === "number" && - Number.isFinite(source.page) && - source.page >= 0; - const MetricBadge = ({ label, value, @@ -142,39 +134,27 @@ export default function SourceCard({ sources = [], onPageClick }: Props) {
{sources.map((src, i) => { const badgeMeta = getConfidenceBadgeMeta( - getPrimarySourceMetric(src), + getPrimarySourceMetric(src) ); - const validPage = hasValidPage(src); return ( { - if (!validPage) return; + onClick={() => onPageClick({ page: src.page + 1, highlightRects: src.highlightRects, - }); - }} - aria-label={ - validPage - ? `Go to source page ${src.page + 1}. Confidence ${badgeMeta.label}` - : `Source from ${src.filename}. Page unavailable. Confidence ${badgeMeta.label}` + }) } + aria-label={`Go to source page ${src.page + 1}. Confidence ${badgeMeta.label}`} > - {validPage ? `p.${src.page + 1}` : "Page N/A"} -{" "} - {badgeMeta.label} + p.{src.page + 1} - {badgeMeta.label} - {sources.map((src, i) => { - const validPage = hasValidPage(src); - - return ( -
-
-
- - {src.filename} - - - {validPage ? `Page ${src.page + 1}` : "Page N/A"} - - - -
- + {sources.map((src, i) => ( +
+
+
+ + {src.filename} + + + Page {src.page + 1} + + +
-

+ onPageClick({ + page: src.page + 1, + highlightRects: src.highlightRects, + }) + } + aria-label={`View source page ${src.page + 1}`} > - {src.text} -

- {src.text.length > EXCERPT_THRESHOLD && ( - - )} + + View +
- ); - })} +

+ {src.text} +

+ {src.text.length > EXCERPT_THRESHOLD && ( + + )} +
+ ))}
)}
diff --git a/frontend/src/components/document/BulkActionBar.tsx b/frontend/src/components/document/BulkActionBar.tsx deleted file mode 100644 index fc77f9d6..00000000 --- a/frontend/src/components/document/BulkActionBar.tsx +++ /dev/null @@ -1,87 +0,0 @@ -"use client"; - -import { Trash2, X, RotateCcw, Loader2 } from "lucide-react"; -import { Button } from "@/components/ui/button"; - -interface BulkActionBarProps { - selectedCount: number; - onDelete: () => void; - onRestore?: () => void; - onClear: () => void; - deleting: boolean; - restoring?: boolean; - mode?: "active" | "trash"; -} - -export default function BulkActionBar({ - selectedCount, - onDelete, - onRestore, - onClear, - deleting, - restoring = false, - mode = "active", -}: BulkActionBarProps) { - if (selectedCount === 0) return null; - - return ( -
- - {selectedCount} selected - - -
- {mode === "trash" && onRestore && ( - - )} - - {mode === "active" && ( - - )} - - -
-
- ); -} diff --git a/frontend/src/components/document/CompareView.tsx b/frontend/src/components/document/CompareView.tsx deleted file mode 100644 index 89ea7d13..00000000 --- a/frontend/src/components/document/CompareView.tsx +++ /dev/null @@ -1,189 +0,0 @@ -"use client"; - -import { useState } from "react"; -import dynamic from "next/dynamic"; -import { X, Columns2, ChevronDown } from "lucide-react"; -import { Button } from "@/components/ui/button"; -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuTrigger, -} from "@/components/ui/dropdown-menu"; -import type { DocInfo } from "@/app/dashboard/page"; - -function PDFViewerSkeleton() { - return ( -
-
-
-
- ); -} - -const PDFViewer = dynamic(() => import("@/components/document/PDFViewer"), { - ssr: false, - loading: () => , -}); - -interface Props { - documents: DocInfo[]; - onClose: () => void; -} - -export default function CompareView({ documents, onClose }: Props) { - const pdfDocs = documents.filter( - (d) => d.status === "ready" && d.original_name.endsWith(".pdf"), - ); - - const [leftDoc, setLeftDoc] = useState(pdfDocs[0] ?? null); - const [rightDoc, setRightDoc] = useState(pdfDocs[1] ?? null); - const [leftPage, setLeftPage] = useState(1); - const [rightPage, setRightPage] = useState(1); - - return ( -
- {/* ── Toolbar ── */} -
- - Compare Documents - -
- { - setLeftDoc(doc); - setLeftPage(1); - }} - /> - - vs - - { - setRightDoc(doc); - setRightPage(1); - }} - /> -
- - -
- - {/* ── Split Panes ── */} -
- {/* Left pane */} -
- {leftDoc ? ( - - ) : ( - - )} -
- - {/* Right pane */} -
- {rightDoc ? ( - - ) : ( - - )} -
-
-
- ); -} - -function DocPicker({ - label, - selected, - options, - exclude, - onSelect, -}: { - label: string; - selected: DocInfo | null; - options: DocInfo[]; - exclude: DocInfo | null; - onSelect: (doc: DocInfo) => void; -}) { - const available = options.filter((d) => d.id !== exclude?.id); - - return ( - - - } - > - {label}: - - {selected?.original_name ?? "Select PDF"} - - - - - {available.length === 0 ? ( - No other PDFs available - ) : ( - available.map((doc) => ( - onSelect(doc)} - className="text-xs truncate" - > - {doc.original_name} - - )) - )} - - - ); -} - -function EmptyPane({ side }: { side: "left" | "right" }) { - return ( -
-
- -

- Select a PDF for the {side} pane -

-

- Use the dropdown above to choose a document -

-
-
- ); -} diff --git a/frontend/src/components/document/DashboardDropOverlay.tsx b/frontend/src/components/document/DashboardDropOverlay.tsx deleted file mode 100644 index 0000a18a..00000000 --- a/frontend/src/components/document/DashboardDropOverlay.tsx +++ /dev/null @@ -1,59 +0,0 @@ -"use client"; - -import { UploadCloud } from "lucide-react"; - -interface DashboardDropOverlayProps { - isDraggingOver: boolean; - dropZoneProps: { - onDragEnter: (e: React.DragEvent) => void; - onDragOver: (e: React.DragEvent) => void; - onDragLeave: (e: React.DragEvent) => void; - onDrop: (e: React.DragEvent) => void; - }; -} - -export default function DashboardDropOverlay({ - isDraggingOver, - dropZoneProps, -}: DashboardDropOverlayProps) { - if (!isDraggingOver) return null; - - return ( -
- {/* Blurred backdrop */} -
- - {/* Animated dashed border card */} -
- {/* Pulsing icon ring */} -
- - - -
- -
-

- Drop files to upload -

-

- PDF, DOCX, TXT, MD supported -

-
-
-
- ); -} diff --git a/frontend/src/components/document/DocumentCard.tsx b/frontend/src/components/document/DocumentCard.tsx deleted file mode 100644 index 5cbcb246..00000000 --- a/frontend/src/components/document/DocumentCard.tsx +++ /dev/null @@ -1,22 +0,0 @@ -"use client"; - -import type { DocInfo } from "@/app/dashboard/page"; -import KeywordTags from "./KeywordTags"; - -interface DocumentCardProps { - document: DocInfo; - isIndexing?: boolean; -} - -export default function DocumentCard({ document, isIndexing }: DocumentCardProps) { - const indexing = - isIndexing ?? - (document.status === "processing" || document.status === "pending"); - - return ( - - ); -} diff --git a/frontend/src/components/document/DocumentSidebar.tsx b/frontend/src/components/document/DocumentSidebar.tsx index 431d721d..a4c158e5 100644 --- a/frontend/src/components/document/DocumentSidebar.tsx +++ b/frontend/src/components/document/DocumentSidebar.tsx @@ -10,25 +10,12 @@ import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Badge } from "@/components/ui/badge"; import { Progress } from "@/components/ui/progress"; -import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { - FileText, - Upload, - UploadCloud, - Trash2, - FileCheck, - Clock, - AlertCircle, - Loader2, - FolderOpen, - Cloud, - RefreshCw, + FileText, Upload, Trash2, FileCheck, Clock, AlertCircle, Loader2, FolderOpen, Cloud, } from "lucide-react"; import { useDropzone } from "react-dropzone"; import { Settings } from "lucide-react"; import DocumentSettings from "./DocumentSettings"; -import DocumentCard from "./DocumentCard"; -import BulkActionBar from "./BulkActionBar"; import { toast } from "sonner"; interface Props { @@ -44,10 +31,7 @@ function DocumentListSkeleton() { return (