From 37f99c73a07ed019ba5e251c38a44ef5ca40a8aa Mon Sep 17 00:00:00 2001 From: Salma Mohamed Date: Sat, 14 Mar 2026 03:24:51 +0200 Subject: [PATCH 1/5] ocr module --- app.py | 3 ++- spendoo/categorization/service.py | 15 ++++++--------- spendoo/core/config.py | 2 +- spendoo/ocr/pipeline.py | 21 +++++++++++++++++++++ spendoo/ocr/routes.py | 22 ++++++++++++++-------- spendoo/ocr/service.py | 27 ++++++++++++++++++++++++--- 6 files changed, 68 insertions(+), 22 deletions(-) create mode 100644 spendoo/ocr/pipeline.py diff --git a/app.py b/app.py index 6a9353a..d842cbd 100644 --- a/app.py +++ b/app.py @@ -1,11 +1,12 @@ from fastapi import FastAPI, APIRouter from spendoo.categorization.routes import router as categorization_router +from spendoo.ocr.routes import router as ocr_router app = FastAPI(title="Spendoo API") api_v1_router = APIRouter(prefix="/api/v1") - +api_v1_router.include_router(ocr_router) api_v1_router.include_router(categorization_router) app.include_router(api_v1_router) diff --git a/spendoo/categorization/service.py b/spendoo/categorization/service.py index 5c5fc33..98c0c1e 100644 --- a/spendoo/categorization/service.py +++ b/spendoo/categorization/service.py @@ -15,28 +15,27 @@ class CategorizationService: def __init__(self): self.llm = LLMClient() - self.model_name = "openai/gpt-oss-20b" + self.model_name = "openai/gpt-oss-120b" def extract(self, text: str): + print("CategorizationService received text:", text) # Debugging log prompt = f""" You are a financial receipt parser. Extract all transaction items from the text below. - For each item return: + For each item try to return: - id (incremental starting from 1 in order of appearance) - item_name - - quantity (if not mentioned assume 1) - - unit_price - - total_price (quantity × unit_price) + - price - category_id (choose one id from the list below) Available categories: {category_block} - Also calculate grand_total (sum of total_price). + Also calculate grand_total or extract it if it's explicitly mentioned in the text. If you can't find a clear grand total, sum up the item prices. If no suitable category exists return null. @@ -47,9 +46,7 @@ def extract(self, text: str): {{ "id": 1, "item_name": "...", - "quantity": 1, - "unit_price": 0, - "total_price": 0, + "price": 0, "category": "...", "category_id": null }} diff --git a/spendoo/core/config.py b/spendoo/core/config.py index 6f8b706..67f548e 100644 --- a/spendoo/core/config.py +++ b/spendoo/core/config.py @@ -5,5 +5,5 @@ class Settings: GROQ_API_KEY: str = os.getenv("GROQ_API_KEY") - + MISTRAL_API_KEY: str = os.getenv("MISTRAL_API_KEY") settings = Settings() \ No newline at end of file diff --git a/spendoo/ocr/pipeline.py b/spendoo/ocr/pipeline.py new file mode 100644 index 0000000..add9955 --- /dev/null +++ b/spendoo/ocr/pipeline.py @@ -0,0 +1,21 @@ +from spendoo.ocr.service import OCRService +from spendoo.categorization.service import CategorizationService +import base64 + +class ReceiptPipeline: + + def __init__(self): + + self.ocr = OCRService() + self.extractor = CategorizationService() + + def process_receipt(self, image_bytes): + + base64_image = base64.b64encode(image_bytes).decode("utf-8") + # Step 1: OCR + receipt_text = self.ocr.extract_text(base64_image) + + # Step 2: LLM extraction + structured_data = self.extractor.extract(receipt_text) + + return structured_data \ No newline at end of file diff --git a/spendoo/ocr/routes.py b/spendoo/ocr/routes.py index 9ded8af..a694aed 100644 --- a/spendoo/ocr/routes.py +++ b/spendoo/ocr/routes.py @@ -1,13 +1,19 @@ -from . import bp -from flask import jsonify +from fastapi import APIRouter, UploadFile, HTTPException +import base64 +from spendoo.ocr.pipeline import ReceiptPipeline +router = APIRouter(prefix="/ocr", tags=["OCR"]) -@bp.route('/', methods=['GET']) -def info(): - return jsonify(module='ocr', status='ok') +pipeline = ReceiptPipeline() +@router.post("/scan") +async def scan_receipt(file: UploadFile): -@bp.route('/health', methods=['GET']) -def health(): - return jsonify(status='ok') + if not file.content_type.startswith("image/"): + raise HTTPException(status_code=400, detail="Invalid file type. Please upload an image.") + + image_bytes = await file.read() + result = pipeline.process_receipt(image_bytes) + + return result \ No newline at end of file diff --git a/spendoo/ocr/service.py b/spendoo/ocr/service.py index a8879a9..72458b3 100644 --- a/spendoo/ocr/service.py +++ b/spendoo/ocr/service.py @@ -1,5 +1,26 @@ -# OCR service layer placeholder +import base64 +from mistralai import Mistral +from spendoo.core.config import settings +from groq import Groq +import base64 -def get_status(): - return {'module': 'ocr', 'status': 'ok'} +class OCRService: + def __init__(self): + self.client = Mistral(api_key=settings.MISTRAL_API_KEY) + + def extract_text(self, base64_image): + + response = self.client.ocr.process( + model="mistral-ocr-latest", + document={ + "type": "image_url", + "image_url": f"data:image/jpeg;base64,{base64_image}" + }, + # table_format="markdown" + ) + + return "\n\n".join( + f"### Page {i+1}\n{response.pages[i].markdown}" + for i in range(len(response.pages)) + ) \ No newline at end of file From ded193b3dd65d162875f29933fed9d4c0cc6f81d Mon Sep 17 00:00:00 2001 From: Salma Mohamed Date: Sat, 14 Mar 2026 20:46:59 +0200 Subject: [PATCH 2/5] minimal fixes --- requirements.txt | 2 +- spendoo/categorization/service.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 152dbdf..df96c3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,4 @@ pytest-flask>=1.2 openai>=1.0,<2 fastapi>=0.68,<1 uvicorn>=0.15,<1 - +mistralai==1.12.4 diff --git a/spendoo/categorization/service.py b/spendoo/categorization/service.py index 98c0c1e..cfa2a1c 100644 --- a/spendoo/categorization/service.py +++ b/spendoo/categorization/service.py @@ -19,7 +19,6 @@ def __init__(self): def extract(self, text: str): - print("CategorizationService received text:", text) # Debugging log prompt = f""" You are a financial receipt parser. From cec9742e88ea9b8b854dea18e02e1236c5cd7c98 Mon Sep 17 00:00:00 2001 From: Joseph Sameh <20220099@stud.fci-cu.edu.eg> Date: Sat, 14 Mar 2026 21:07:03 +0200 Subject: [PATCH 3/5] Expose MISTRAL_API_KEY to Docker runs Add MISTRAL_API_KEY (from repo secrets) to the docker run commands in .github/workflows/build-backend.yml and .github/workflows/deploy.yml so the backend container receives the Mistral API key during CI/test and production deployment. --- .github/workflows/build-backend.yml | 2 +- .github/workflows/deploy.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 9843e96..39a5fff 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -25,7 +25,7 @@ jobs: run: | # Start container in background docker run -d --name spendoo-test \ - -e GROQ_API_KEY=${{ secrets.GROQ_API_KEY }} -e SPENDOO_DEPLOY=${{ secrets.SPENDOO_DEPLOY }} -e SPENDOO_ALLOWED_IP=${{ secrets.SPENDOO_ALLOWED_IP }} -p 8000:8000 spendoo-ai-backend + -e GROQ_API_KEY=${{ secrets.GROQ_API_KEY }} -e MISTRAL_API_KEY=${{ secrets.MISTRAL_API_KEY }} -e SPENDOO_DEPLOY=${{ secrets.SPENDOO_DEPLOY }} -e SPENDOO_ALLOWED_IP=${{ secrets.SPENDOO_ALLOWED_IP }} -p 8000:8000 spendoo-ai-backend # Wait for FastAPI to start sleep 10 # Check health endpoint diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7a67076..203d828 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -30,5 +30,5 @@ jobs: docker stop spendoo-ai-container || true docker rm spendoo-ai-container || true docker run -d --restart=always --name spendoo-ai-container -p 8000:8000 \ - -e GROQ_API_KEY="${{ secrets.GROQ_API_KEY }}" -e SPENDOO_DEPLOY="${{ secrets.SPENDOO_DEPLOY }}" -e SPENDOO_ALLOWED_IP="${{ secrets.SPENDOO_ALLOWED_IP }}" josephsameh/spendoo-ai-backend:latest + -e GROQ_API_KEY="${{ secrets.GROQ_API_KEY }}" -e MISTRAL_API_KEY="${{ secrets.MISTRAL_API_KEY }}" -e SPENDOO_DEPLOY="${{ secrets.SPENDOO_DEPLOY }}" -e SPENDOO_ALLOWED_IP="${{ secrets.SPENDOO_ALLOWED_IP }}" josephsameh/spendoo-ai-backend:latest docker image prune -f \ No newline at end of file From dae0ec7859795879cc624e1678026f5a614930ff Mon Sep 17 00:00:00 2001 From: Joseph Sameh Fouad <20220099@stud.fci-cu.edu.eg> Date: Sat, 14 Mar 2026 21:11:59 +0200 Subject: [PATCH 4/5] Increase sleep duration before health check Increased wait time for FastAPI to start from 10 to 30 seconds. --- .github/workflows/build-backend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 39a5fff..ff588fe 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -27,7 +27,7 @@ jobs: docker run -d --name spendoo-test \ -e GROQ_API_KEY=${{ secrets.GROQ_API_KEY }} -e MISTRAL_API_KEY=${{ secrets.MISTRAL_API_KEY }} -e SPENDOO_DEPLOY=${{ secrets.SPENDOO_DEPLOY }} -e SPENDOO_ALLOWED_IP=${{ secrets.SPENDOO_ALLOWED_IP }} -p 8000:8000 spendoo-ai-backend # Wait for FastAPI to start - sleep 10 + sleep 30 # Check health endpoint curl --fail http://localhost:8000/docs # Stop and remove container From f2408aba162fd2102513d477a201b24a8d5e67cb Mon Sep 17 00:00:00 2001 From: Joseph Sameh <20220099@stud.fci-cu.edu.eg> Date: Sat, 14 Mar 2026 21:18:10 +0200 Subject: [PATCH 5/5] Add groq and python-multipart to requirements Update requirements.txt to include groq>=0.1,<1 and python-multipart>=0.0.5,<1. These dependencies enable GROQ query support and multipart/form-data handling (e.g., file uploads) for the application. --- .github/workflows/build-backend.yml | 2 +- requirements.txt | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index ff588fe..39a5fff 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -27,7 +27,7 @@ jobs: docker run -d --name spendoo-test \ -e GROQ_API_KEY=${{ secrets.GROQ_API_KEY }} -e MISTRAL_API_KEY=${{ secrets.MISTRAL_API_KEY }} -e SPENDOO_DEPLOY=${{ secrets.SPENDOO_DEPLOY }} -e SPENDOO_ALLOWED_IP=${{ secrets.SPENDOO_ALLOWED_IP }} -p 8000:8000 spendoo-ai-backend # Wait for FastAPI to start - sleep 30 + sleep 10 # Check health endpoint curl --fail http://localhost:8000/docs # Stop and remove container diff --git a/requirements.txt b/requirements.txt index 902c9ad..f246f9f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,5 @@ fastapi>=0.68,<1 uvicorn>=0.15,<1 mistralai==1.12.4 python-dotenv>=0.19,<1 +groq>=0.1,<1 +python-multipart>=0.0.5,<1 \ No newline at end of file