[ShanaBoo] Resilient background job retry & monitoring

.env.example

@@ -1,5 +1,6 @@
-DATABASE_URL="postgresql+psycopg2://finmind:finmind@postgres:5432/finmind"
-REDIS_URL="redis://redis:6379/0"
+DATABASE_URL=sqlite:///finmind.db
+JWT_SECRET_KEY=super-secret
+REDIS_URL=redis://localhost:6379/0
 POSTGRES_USER="finmind"
 POSTGRES_PASSWORD="finmind"
 POSTGRES_DB="finmind"

README.md

@@ -44,21 +44,28 @@ flowchart LR
 ```
 
 ## PostgreSQL Schema (DDL)
-See `backend/app/db/schema.sql`. Key tables:
-- users, categories, expenses, bills, reminders
-- ad_impressions, subscription_plans, user_subscriptions
-- refresh_tokens (optional if rotating), audit_logs
+    JWT[PyJWT]
+    AI[Insights Service]
+    SCH[Scheduler/APScheduler]
+    RED[Redis]
+  end
 
-## Redis Caching Policy
+  subgraph Data
+    PG[(PostgreSQL)]
+    RD[(Redis)]
+  end
+
+  subgraph ThirdParty
 - Keys
   - `user:{id}:monthly_summary:{yyyy-mm}` — 30 min TTL
   - `user:{id}:categories` — 24h TTL
-  - `user:{id}:upcoming_bills` — 15 min TTL
-  - `insights:{id}` — 24h TTL (invalidate on new expense/bill)
-- Invalidation
-  - On expense/bill create/update/delete -> delete affected monthly_summary, upcoming_bills, insights
-- Rate limiting (optional): `rl:{userId}:{endpoint}:{minute}` with short TTL
+  end
 
+  A -->|HTTPS| CDN --> API
+  API -->|Job Queue| RED
+  API -->|ORM| PG
+  API -->|Cache| RD
+  API -->|JWT verify| JWT
 ## API Endpoints
 OpenAPI: `backend/app/openapi.yaml`
 - Auth: `/auth/register`, `/auth/login`, `/auth/refresh`

backend/app/__init__.py

@@ -0,0 +1,11 @@
+from flask import Flask
+from .config import Config
+from .extensions import init_extensions, db, jwt, scheduler
+from .routes import register_routes
+
+def create_app():
+    app = Flask(__name__)
+    app.config.from_object(Config)
+    init_extensions(app)
+    register_routes(app)
+    return app

backend/app/config.py

@@ -0,0 +1,10 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+class Config:
+    SQLALCHEMY_DATABASE_URI = os.getenv('DATABASE_URL', 'sqlite:///finmind.db')
+    SQLALCHEMY_TRACK_MODIFICATIONS = False
+    JWT_SECRET_KEY = os.getenv('JWT_SECRET_KEY', 'super-secret')
+    REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0')

backend/app/extensions.py

@@ -0,0 +1,23 @@
+from flask_sqlalchemy import SQLAlchemy
+from flask_jwt_extended import JWTManager
+from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.executors.pool import ThreadPoolExecutor
+from apscheduler.jobstores.redis import RedisJobStore
+import logging
+
+db = SQLAlchemy()
+jwt = JWTManager()
+
+scheduler = BackgroundScheduler(
+    jobstores={'default': RedisJobStore(host='localhost', port=6379, db=0)},
+    executors={'default': ThreadPoolExecutor(20)},
+    job_defaults={'coalesce': False, 'max_instances': 3},
+    timezone='UTC'
+)
+
+def init_extensions(app):
+    db.init_app(app)
+    jwt.init_app(app)
+    scheduler.start()
+    app.logger.setLevel(logging.INFO)
+    app.logger.addHandler(logging.StreamHandler())

backend/app/models.py

@@ -0,0 +1,13 @@
+from flask_sqlalchemy import SQLAlchemy
+from datetime import datetime
+
+db = SQLAlchemy()
+
+class Reminder(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    user_id = db.Column(db.Integer, nullable=False)
+    message = db.Column(db.String(255), nullable=False)
+    due_date = db.Column(db.DateTime, nullable=False)
+    sent = db.Column(db.Boolean, default=False)
+    created_at = db.Column(db.DateTime, default=datetime.utcnow)
+    updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)

backend/app/routes/__init__.py

@@ -0,0 +1,16 @@
+from flask import Blueprint
+from . import auth, expenses, bills, reminders, insights
+from ..extensions import scheduler
+
+def register_routes(app):
+    app.register_blueprint(auth.bp)
+    app.register_blueprint(expenses.bp)
+    app.register_blueprint(bills.bp)
+    app.register_blueprint(reminders.bp)
+    app.register_blueprint(insights.bp)
+
+    # Example job registration
+    @scheduler.scheduled_job('interval', id='reminder_job', minutes=1)
+    def reminder_job():
+        app.logger.info("Running reminder job...")
+        # Add job logic here

backend/app/routes/reminders.py

@@ -0,0 +1,43 @@
+from flask import Blueprint, jsonify, request
+from ..extensions import db
+from ..models import Reminder
+from ..extensions import scheduler
+
+bp = Blueprint('reminders', __name__, url_prefix='/reminders')
+
+@bp.route('/run', methods=['POST'])
+def run_reminders():
+    # Logic to manually trigger reminders
+    app.logger.info('Manually running reminders...')
+    # Logic to send reminders
+    return jsonify({"message": "Reminders triggered"}), 200
+
+@bp.route('/', methods=['GET'])
+def get_reminders():
+    reminders = Reminder.query.all()
+    return jsonify([reminder.to_dict() for reminder in reminders])
+
+@bp.route('/', methods=['POST'])
+def create_reminder():
+    data = request.get_json()
+    reminder = Reminder(name=data['name'], due_date=data['due_date'], channel=data['channel'])
+    db.session.add(reminder)
+    db.session.commit()
+    return jsonify(reminder.to_dict()), 201
+
+@bp.route('/<int:id>', methods=['PUT'])
+def update_reminder(id):
+    data = request.get_json()
+    reminder = Reminder.query.get_or_404(id)
+    reminder.name = data['name']
+    reminder.due_date = data['due_date']
+    reminder.channel = data['channel']
+    db.session.commit()
+    return jsonify(reminder.to_dict())
+
+@bp.route('/<int:id>', methods=['DELETE'])
+def delete_reminder(id):
+    reminder = Reminder.query.get_or_404(id)
+    db.session.delete(reminder)
+    db.session.commit()
+    return jsonify({"message": "Reminder deleted"}), 200

deploy/docker-compose.yml

@@ -0,0 +1,25 @@
+version: '3.8'
+
+services:
+  redis:
+    image: redis:latest
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis-data:/data
+
+  backend:
+    build: ../backend
+    ports:
+      - "5000:5000"
+    environment:
+      - DATABASE_URL=postgresql://user:password@db:5432/finmind
+      - REDIS_URL=redis://redis:6379/0
+      - JWT_SECRET_KEY=super-secret
+    depends_on:
+      - db
+      - redis
+
+volumes:
+  db-data:
+  redis-data:

finmind/jobs/__init__.py

@@ -0,0 +1,8 @@
+"""Background job execution module with retry and monitoring capabilities."""
+
+from finmind.jobs.executor import JobExecutor
+from finmind.jobs.retry import RetryPolicy, ExponentialBackoff
+from finmind.jobs.monitor import JobMonitor, JobStatus
+from finmind.jobs.decorators import resilient_job
+
+__all__ = ["JobExecutor", "RetryPolicy", "ExponentialBackoff", "JobMonitor", "JobStatus", "resilient_job"]

scripts/test_scheduler.py

@@ -0,0 +1,16 @@
+import unittest
+from app.extensions import scheduler
+from app import create_app
+
+class TestScheduler(unittest.TestCase):
+    def setUp(self):
+        self.app = create_app()
+        self.app_context = self.app.app_context()
+        self.app_context.push()
+
+    def tearDown(self):
+        self.app_context.pop()
+
+    def test_scheduler_running(self):
+        self.assertTrue(scheduler.running)
+        self.assertEqual(len(scheduler.get_jobs()), 1)