Track runs with failed and successful modules

app/app_runs.py

@@ -0,0 +1,120 @@
+import logging
+from contextlib import asynccontextmanager
+
+from typing import cast
+
+import datetime
+
+import uvicorn
+
+from fastapi import BackgroundTasks, FastAPI, HTTPException, status
+from fastapi.responses import PlainTextResponse
+from fastapi.routing import APIRoute
+
+from app import __version__, db
+
+logger = logging.getLogger("multiqc_api")
+
+logger.info("Starting MultiQC API run logger service")
+
+# Add timestamp to the uvicorn logger
+for h in logging.getLogger("uvicorn.access").handlers:
+    h.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
+
+
+@asynccontextmanager
+async def lifespan(_: FastAPI):
+    yield
+
+
+app = FastAPI(
+    title="MultiQC API",
+    description="MultiQC API service, providing run-time information about available "
+    ""
+    ""
+    ""
+    ""
+    ""
+    ""
+    ""
+    ""
+    "updates.",
+    version=__version__,
+    license_info={
+        "name": "Source code available under the MIT Licence",
+        "url": "https://github.com/MultiQC/api.multiqc.info/blob/main/LICENSE",
+    },
+    lifespan=lifespan,
+)
+
+db.create_db_and_tables()
+
+
+@app.get("/run")
+async def run(
+    background_tasks: BackgroundTasks,
+    duration: str,
+    modules: str,
+    modules_failed: str,
+    version_multiqc: str,
+    version_python: str,
+    operating_system: str = "",
+    is_docker: str = "",
+    is_singularity: str = "",
+    is_conda: str = "",
+    is_ci: str = "",
+):
+    """
+    Log the modules run by MultiQC
+    """
+    timestamp = datetime.datetime.now().isoformat(timespec="microseconds")
+    background_tasks.add_task(
+        db.log_run,
+        timestamp,
+        duration=duration,
+        modules=modules,
+        modules_failed=modules_failed,
+        version_multiqc=version_multiqc,
+        version_python=version_python,
+        operating_system=operating_system,
+        is_docker=is_docker,
+        is_singularity=is_singularity,
+        is_conda=is_conda,
+        is_ci=is_ci,
+        user_ip=None,
+    )
+
+
+@app.get("/")
+async def index(_: BackgroundTasks):
+    """
+    Root endpoint for the API.
+
+    Returns a list of available endpoints.
+    """
+    routes = [cast(APIRoute, r) for r in app.routes]
+    return {
+        "message": "Welcome to the MultiQC runs logging service",
+        "available_endpoints": [
+            {"path": route.path, "name": route.name} for route in routes if route.name != "swagger_ui_redirect"
+        ],
+    }
+
+
+@app.get("/health")
+async def health():
+    """
+    Health check endpoint. Checks if the visits table contains records
+    in the past 15 minutes.
+    """
+    try:
+        visits = db.get_visit_stats(start=datetime.datetime.now() - datetime.timedelta(minutes=15))
+    except Exception as e:
+        raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e))
+    if not visits:
+        raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="No recent visits found")
+    return PlainTextResponse(content=str(len(visits)))
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8000)

app/db.py

@@ -1,4 +1,6 @@
 """Functions to interact with the database."""
+
+import logging
 from typing import Optional, Sequence
 
 import os
@@ -9,10 +11,14 @@
 
 from sqlmodel import create_engine, Field, select, Session, SQLModel
 
+from app.utils import strtobool
+
 sql_url = os.getenv("DATABASE_URL")
 assert sql_url is not None, sql_url
 engine = create_engine(sql_url)
 
+logger = logging.getLogger(__name__)
+
 
 class VisitStats(SQLModel, table=True):
     """
@@ -37,6 +43,114 @@ class VisitStats(SQLModel, table=True):
     count: int
 
 
+class User(SQLModel, table=True):
+    __tablename__ = "multiqc_api_user"
+
+    id: int = Field(primary_key=True)
+    ip: str
+
+
+class Run(SQLModel, table=True):
+    __tablename__ = "multiqc_api_run"
+
+    id: int = Field(primary_key=True)
+    user_id: Optional[int] = Field(default=None, foreign_key="multiqc_api_user.id")
+    timestamp: datetime.datetime
+    duration_seconds: int
+    version_multiqc: str
+    version_python: str
+    operating_system: str
+    is_docker: bool
+    is_singularity: bool
+    is_conda: bool
+    is_ci: bool
+
+
+class Module(SQLModel, table=True):
+    __tablename__ = "multiqc_api_module"
+
+    id: int = Field(primary_key=True)
+    name: str
+
+
+class SuccessRunToModule(SQLModel, table=True):
+    __tablename__ = "multiqc_api_success_run_to_module"
+
+    id: int = Field(primary_key=True)
+    run_id: int = Field(foreign_key="multiqc_api_run.id")
+    module_id: int = Field(foreign_key="multiqc_api_module.id")
+
+
+class FailureRunToModule(SQLModel, table=True):
+    __tablename__ = "multiqc_api_failure_run_to_module"
+
+    id: int = Field(primary_key=True)
+    run_id: int = Field(foreign_key="multiqc_api_run.id")
+    module_id: int = Field(foreign_key="multiqc_api_module.id")
+
+
+def log_run(
+    timestamp_str: str,
+    duration: str,
+    modules: str,
+    modules_failed: str,
+    version_multiqc: str,
+    version_python: str,
+    operating_system: str,
+    is_docker: str,
+    is_singularity: str,
+    is_conda: str,
+    is_ci: str,
+    user_ip: Optional[str] = None,
+) -> None:
+    """Log a run of MultiQC."""
+    with Session(engine) as session:
+        try:
+            duration_seconds = int(duration)
+        except ValueError:
+            logger.warning(f"Could not parse duration: {duration}")
+            duration_seconds = -1
+
+        timestamp = datetime.datetime.fromisoformat(timestamp_str)
+
+        run = Run(
+            timestamp=timestamp,
+            duration_seconds=duration_seconds,
+            version_multiqc=version_multiqc,
+            version_python=version_python,
+            operating_system=operating_system,
+            is_docker=strtobool(is_docker),
+            is_singularity=strtobool(is_singularity),
+            is_conda=strtobool(is_conda),
+            is_ci=strtobool(is_ci),
+        )
+        session.add(run)
+        session.commit()
+        for _mod in modules.split(","):
+            module = session.exec(select(Module).where(Module.name == _mod)).first()
+            if not module:
+                module = Module(name=_mod)
+                session.add(module)
+                session.commit()
+            session.add(SuccessRunToModule(run_id=run.id, module_id=module.id))
+        for _mod in modules_failed.split(","):
+            module = session.exec(select(Module).where(Module.name == _mod)).first()
+            if not module:
+                module = Module(name=_mod)
+                session.add(module)
+                session.commit()
+            session.add(FailureRunToModule(run_id=run.id, module_id=module.id))
+        session.add(run)
+
+        if user_ip:
+            user = User(ip=user_ip)
+            session.add(user)
+            session.commit()
+            run.user_id = user.id
+
+        session.commit()
+
+
 class DownloadStats(SQLModel, table=True):
     """
     Daily download statistics.
@@ -129,4 +243,3 @@ def insert_download_stats(df: pd.DataFrame) -> pd.DataFrame:
                 session.add(new_entry)
         session.commit()
     return df
-

docker-compose.yml

@@ -1,10 +1,44 @@
 version: "3.8"
 services:
-  web:
-    container_name: multiqc_api
+  visits:
+    container_name: multiqc_api_visits
     restart: always
     build: .
-    command: uvicorn app.main:app ${UVICORN_RELOAD} --host 0.0.0.0
+    command: uvicorn app.app_visits:app ${UVICORN_RELOAD} --host 0.0.0.0
+    volumes:
+      - .:/code
+    ports:
+      - "8008:8000"
+    environment:
+      # Set in .env
+      GITHUB_TOKEN: $GITHUB_TOKEN
+      # Matches the "db" service below
+      DATABASE_URL: mysql+pymysql://root:1@db:3306/multiqc
+    depends_on:
+      wait-for-db:
+        condition: service_completed_successfully
+  downloads:
+    container_name: multiqc_api_downloads
+    restart: always
+    build: .
+    command: uvicorn app.app_downloads:app ${UVICORN_RELOAD} --host 0.0.0.0
+    volumes:
+      - .:/code
+    ports:
+      - "8008:8000"
+    environment:
+      # Set in .env
+      GITHUB_TOKEN: $GITHUB_TOKEN
+      # Matches the "db" service below
+      DATABASE_URL: mysql+pymysql://root:1@db:3306/multiqc
+    depends_on:
+      wait-for-db:
+        condition: service_completed_successfully
+  runs:
+    container_name: multiqc_api_runs
+    restart: always
+    build: .
+    command: uvicorn app.app_runs:app ${UVICORN_RELOAD} --host 0.0.0.0
     volumes:
       - .:/code
     ports:
@@ -14,7 +48,6 @@ services:
       GITHUB_TOKEN: $GITHUB_TOKEN
       # Matches the "db" service below
       DATABASE_URL: mysql+pymysql://root:1@db:3306/multiqc
-      LOGZIO_TOKEN: $LOGZIO_TOKEN
     depends_on:
       wait-for-db:
         condition: service_completed_successfully