feat: quivr core 0.1 (QuivrHQ#2970)

# Description


# Testing backend 

## Docker setup
1. Copy `.env.example` to `.env`. Some env variables were added :
EMBEDDING_DIM
2. Apply supabase migratrions : 
```sh
supabase stop
supabase db reset
supabase start
```
3. Start backend containers
```
make dev
```
## Local setup 
You can also run backend without docker.
1. Install [`rye`](https://rye.astral.sh/guide/installation/). Choose
the managed python version and set the version to 3.11
2. Run the following: 
```
cd quivr/backend
rye sync
```
3. Source `.venv` virtual env : `source .venv/bin/activate`
4. Run the backend, make sure you are running redis and supabase
API: 
```
LOG_LEVEL=debug uvicorn quivr_api.main:app --log-level debug --reload --host 0.0.0.0 --port 5050 --workers 1
```
Worker: 
```
LOG_LEVEL=debug celery -A quivr_worker.celery_worker worker -l info -E --concurrency 1
```
Notifier: 
```
LOG_LEVEL=debug python worker/quivr_worker/celery_monitor.py
```

---------

Co-authored-by: chloedia <[email protected]>
Co-authored-by: aminediro <[email protected]>
Co-authored-by: Antoine Dewez <[email protected]>
Co-authored-by: Chloé Daems <[email protected]>
Co-authored-by: Zewed <[email protected]>

.env.example

@@ -18,7 +18,7 @@ NEXT_PUBLIC_SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzd
 NEXT_PUBLIC_CMS_URL=https://cms.quivr.app
 NEXT_PUBLIC_FRONTEND_URL=http://localhost:*
 NEXT_PUBLIC_AUTH_MODES=password
-NEXT_PUBLIC_SHOW_TOKENS=false 
+NEXT_PUBLIC_SHOW_TOKENS=false
 #NEXT_PUBLIC_PROJECT_NAME=<project-name>
 
 
@@ -39,6 +39,7 @@ CELERY_BROKER_URL=redis://redis:6379/0
 CELEBRY_BROKER_QUEUE_NAME=quivr-preview.fifo
 QUIVR_DOMAIN=http://localhost:3000/
 BACKEND_URL=http://localhost:5050
+EMBEDDING_DIM=1536
 #COHERE_API_KEY=CHANGE_ME
 DEACTIVATE_STRIPE=true
 

.github/workflows/backend-core-tests.yml

@@ -20,20 +20,18 @@ jobs:
           - 9998:9998
 
     steps:
-      - name: Checkout code
+      - name: 👀 Checkout code
         uses: actions/checkout@v2
 
-      - name: Set up Python
-        uses: actions/setup-python@v2
+      - name: 🔨 Install the latest version of rye
+        uses: eifinger/setup-rye@v4
         with:
-          python-version: "3.11"
-
-      - name: Install dependencies
+          enable-cache: true
+          working-directory: backend
+      - name: 🔄 Sync dependencies
         run: |
-          cd backend/core
-          python -m pip install --upgrade pip
-          pip install poetry
-          poetry install -E base --with dev,test
+          cd backend
+          UV_INDEX_STRATEGY=unsafe-first-match rye sync --no-lock
 
       - name: Run tests
         env:
@@ -42,6 +40,7 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y libmagic-dev poppler-utils libreoffice tesseract-ocr  pandoc
-          tesseract --version
-          cd backend/core
-          poetry run tox
+          cd backend
+          rye run python -c "from unstructured.nlp.tokenize import download_nltk_packages; download_nltk_packages()"
+          rye run python -c "import nltk;nltk.download('punkt_tab'); nltk.download('averaged_perceptron_tagger_eng')"
+          rye test -p quivr-core

.github/workflows/backend-tests.yml

@@ -1,74 +1,68 @@
-# name: Backend Tests
+name: Run Tests API and Worker
 
-# on:
-#   push:
-#     branches: [main]
-#     paths:
-#       - 'backend/**'
-#   pull_request:
-#     branches: [main]
-#     paths:
-#       - 'backend/**'
-#   workflow_dispatch:
-#     inputs:
-#       logLevel:
-#         description: 'Log level'
-#         required: true
-#         default: 'warning'
-#         type: choice
-#         options:
-#         - info
-#         - warning
-#         - debug
-#       tags:
-#         description: 'Test scenario tags'
-#         required: false
-#         type: boolean
-#       environment:
-#         description: 'Environment to run tests against'
-#         type: environment
-#         required: false
+on:
+  pull_request:
+    paths:
+      - "backend/**"
+  workflow_dispatch:
 
-# jobs:
-#   build:
-#     runs-on: ubuntu-latest
-#     defaults:
-#       run:
-#         working-directory: ./backend
-#     environment: preview
-#     strategy:
-#       matrix:
-#         python-version: ["3.11"]
+jobs:
+  test:
+    runs-on: ubuntu-latest
 
-#     steps:
-#       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
-#       - name: Set up Python ${{ matrix.python-version }}
-#         uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4
-#         with:
-#           python-version: ${{ matrix.python-version }}
-#           cache: "pip"
-#       - name: Install system dependencies
-#         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
-#       - name: Install dependencies
-#         run: |
-#           python -m pip install --upgrade pip
-#           pip install pytest pytest-emoji pytest-md pytest-mock
-#           pip install pyright
-#           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-#       - name: Run pytest
-#         uses: pavelzw/pytest-action@510c5e90c360a185039bea56ce8b3e7e51a16507 # v2
-#         with:
-#           verbose: true
-#           emoji: true
-#           job-summary: true
-#           click-to-expand: false
-#           report-title: 'Test Report'
-#         env:
-#           SUPABASE_URL: ${{secrets.SUPABASE_URL}}
-#           SUPABASE_SERVICE_KEY: ${{secrets.SUPABASE_SERVICE_KEY}}
-#           OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
-#           ANTHROPIC_API_KEY: ${{secrets.ANTHROPIC_API_KEY}}
-#           JWT_SECRET_KEY: ${{secrets.JWT_SECRET_KEY}}
-#           CI_TEST_API_KEY: ${{secrets.CI_TEST_API_KEY}}
-#           CELERY_BROKER_URL: ${{secrets.CELERY_BROKER_URL}}
+    steps:
+      - name: 👀 Checkout code
+        uses: actions/checkout@v2
 
+      - name: 🔨 Install the latest version of rye
+        uses: eifinger/setup-rye@v4
+        with:
+          enable-cache: true
+          working-directory: backend
+
+      - name: 🔄 Sync dependencies
+        run: |
+          cd backend
+          UV_INDEX_STRATEGY=unsafe-first-match rye sync --no-lock
+
+      - name: 🚤 Install Supabase CLI
+        run: |
+          ARCHITECTURE=$(uname -m)
+          if [ "$ARCHITECTURE" = "x86_64" ]; then
+            wget https://github.com/supabase/cli/releases/download/v1.163.6/supabase_1.163.6_linux_amd64.deb
+            sudo dpkg -i supabase_1.163.6_linux_amd64.deb
+          elif [ "$ARCHITECTURE" = "aarch64" ]; then
+            wget https://github.com/supabase/cli/releases/download/v1.163.6/supabase_1.163.6_linux_arm64.deb
+            sudo dpkg -i supabase_1.163.6_linux_arm64.deb
+          fi
+
+      - name: 😭 Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libmagic-dev poppler-utils libreoffice tesseract-ocr pandoc
+
+      - name: Install dependencies and run tests
+        env:
+          OPENAI_API_KEY: this-is-a-fake-openai-api-key
+          SUPABASE_URL: http://localhost:54321
+          SUPABASE_SERVICE_KEY: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MzgxMjk5Nn0.EGIM96RAZx35lJzdJsyH-qQwv8Hdp7fsn3W0YpN81IU
+          PG_DATABASE_URL: postgresql://postgres:postgres@localhost:54322/postgres
+          PG_DATABASE_ASYNC_URL: postgresql+asyncpg://postgres:postgres@localhost:54322/postgres
+          ANTHROPIC_API_KEY: null
+          JWT_SECRET_KEY: super-secret-jwt-token-with-at-least-32-characters-long
+          AUTHENTICATE: true
+          TELEMETRY_ENABLED: true
+          CELERY_BROKER_URL: redis://redis:6379/0
+          CELEBRY_BROKER_QUEUE_NAME: quivr-preview.fifo
+          QUIVR_DOMAIN: http://localhost:3000/
+          BACKEND_URL: http://localhost:5050
+          EMBEDDING_DIM: 1536
+        run: |
+          cd backend
+          sed -i 's/enabled = true/enabled = false/g' supabase/config.toml
+          sed -i '/\[db\]/,/\[.*\]/s/enabled = false/enabled = true/' supabase/config.toml
+          sed -i '/\[storage\]/,/\[.*\]/s/enabled = false/enabled = true/' supabase/config.toml
+          supabase start
+          rye run python -c "from unstructured.nlp.tokenize import download_nltk_packages; download_nltk_packages()"
+          rye run python -c "import nltk;nltk.download('punkt_tab'); nltk.download('averaged_perceptron_tagger_eng')"
+          rye test -p quivr-api -p quivr-worker

.gitignore

@@ -1,6 +1,7 @@
 docker-compose.override.yml
 secondbrain/
 .env
+env.sh
 .streamlit/secrets.toml
 **/*.pyc
 toto.txt
@@ -18,6 +19,8 @@ backend/.python-version
 **/.pnp
 .pnp.js
 
+Pipfile
+
 # testing
 **/coverage
 
@@ -59,7 +62,7 @@ backend/core/local_models/*
 
 ## scripts
 package-lock.json
-backend/celerybeat-schedule
+celerybeat-schedule
 frontend/public/robots.txt
 frontend/public/sitemap*
 
@@ -98,3 +101,4 @@ backend/core/examples/chatbot/.chainlit/translations/en-US.json
 
 # Tox
 .tox
+Pipfile

.run_tests.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+set -e
+
+## TESTS SUITES
+test_suites=(
+        "Backend Core:cd backend/core && tox -p auto"
+        "Worker:cd backend && pytest worker"
+        "API:cd backend && pytest api"
+    )
+
+# Check if gum is installed
+if ! command -v gum &>/dev/null; then
+    echo "gum is not installed. Please install it with 'brew install gum'."
+    exit 1
+fi
+
+root_dir=$(pwd)
+
+# Function to check if Tika server is running
+check_tika_server() {
+    if nc -z localhost 9998 >/dev/null 2>&1; then
+        return 0
+    else
+        gum style --foreground 196 "Error: Tika server is not running on port 9998."
+        gum style --foreground 226 "Please start the Tika server before running the tests."
+        gum style --foreground 226 "Run 'docker run -d -p 9998:9998 apache/tika' to start the Tika server."
+        return 1
+    fi
+}
+
+# select test suites to run, either all or one of the following
+get_test_suites_to_run() {
+    gum style --bold "Select test suites to run:"
+    options=("All" "${test_suites[@]%%:*}")
+    selected=$(gum choose "${options[@]}")
+    if [[ "$selected" == "All" ]]; then
+        gum style --bold "Running all test suites"
+    else
+        # Find the matching test suite
+        for suite in "${test_suites[@]}"; do
+            if [[ "${suite%%:*}" == "$selected" ]]; then
+                test_suites=("$suite")
+                break
+            fi
+        done
+    fi
+}
+
+# Function to run a single test suite
+run_test_suite() {
+    local suite_name=$1
+    local command=$2
+    local exit_code
+
+    gum style --border normal --border-foreground 99 --padding "1 2" --bold "$suite_name Tests"
+    eval "$command"
+    exit_code=$?
+    cd "$root_dir"
+
+    if [ $exit_code -eq 0 ]; then
+        gum style --foreground 46 "$suite_name Tests: PASSED"
+    else
+        gum style --foreground 196 "$suite_name Tests: FAILED"
+    fi
+
+    return $exit_code
+}
+
+run_tests() {
+    get_test_suites_to_run
+    # gum spin --spinner dot --title "Running tests..." -- sleep 1
+
+    local all_passed=true
+    local results=()
+
+    for suite in "${test_suites[@]}"; do
+        IFS=':' read -r suite_name suite_command <<< "$suite"
+        if ! run_test_suite "$suite_name" "$suite_command"; then
+            all_passed=false
+        fi
+        results+=("$suite_name:$?")
+    done
+
+    # Print summary of test results
+    gum style --border double --border-foreground 99 --padding "1 2" --bold "Test Summary"
+    for result in "${results[@]}"; do
+        IFS=':' read -r suite_name exit_code <<< "$result"
+        if [ "$exit_code" -eq 0 ]; then
+            gum style --foreground 46 "✓ $suite_name: PASSED"
+        else
+            gum style --foreground 196 "✗ $suite_name: FAILED"
+        fi
+    done
+
+    # Return overall exit code
+    $all_passed
+}
+
+# Main execution
+if check_tika_server; then
+    run_tests
+    exit $?
+else
+    exit 1
+fi

Makefile

@@ -1,38 +1,34 @@
-test:
-	pytest backend/
-
-dev-build:
-	DOCKER_BUILDKIT=1 docker compose -f docker-compose.dev.yml build backend-core
-	DOCKER_BUILDKIT=1 docker compose -f docker-compose.dev.yml up --build
-
-dev:
-	DOCKER_BUILDKIT=1 docker compose -f docker-compose.dev.yml up
+.DEFAULT_TARGET=help
 
+## help: Display list of commands
+.PHONY: help
+help:
+	@echo "Available commands:"
+	@sed -n 's|^##||p' $(MAKEFILE_LIST) | column -t -s ':' | sed -e 's|^| |'
 
-dev-saas:
-	docker compose -f docker-compose-dev-saas-supabase.yml build backend-core
-	docker compose -f docker-compose-dev-saas-supabase.yml up --build
 
-dev-saas-back:
-	docker compose -f docker-compose-dev-only-back-saas-supabase.yml build backend-core
-	docker compose -f docker-compose-dev-only-back-saas-supabase.yml up --build backend-core
+## dev: Start development environment
+.PHONY: dev
+dev:
+	DOCKER_BUILDKIT=1 docker compose -f docker-compose.dev.yml up --build
 
-dev-stan:
-	docker compose -f docker-compose-no-frontend.dev.yml up --build 
+dev-build:
+	DOCKER_BUILDKIT=1 docker compose -f docker-compose.dev.yml build --no-cache
+	DOCKER_BUILDKIT=1 docker compose -f docker-compose.dev.yml up
 
+## prod: Build and start production environment
+.PHONY: prod
 prod:
-	docker compose build backend-core
 	docker compose -f docker-compose.yml up --build
 
-test-type:
-	@if command -v python3 &>/dev/null; then \
-		python3 -m pyright; \
-	else \
-		python -m pyright; \
-	fi
-
+## front: Build and start frontend
+.PHONY: front
 front:
-	cd frontend  && yarn build && yarn start
+	cd frontend  && yarn && yarn build && yarn start
 
+## test: Run tests
+.PHONY: test
 test:
-	cd backend/core && ./scripts/run_tests.sh
+	# Ensure dependencies are installed with dev and test extras
+	# poetry install --with dev,test && brew install tesseract pandoc libmagic
+	./.run_tests.sh