diff --git a/app/app_runs.py b/app/app_runs.py new file mode 100644 index 0000000..7bd5b72 --- /dev/null +++ b/app/app_runs.py @@ -0,0 +1,120 @@ +import logging +from contextlib import asynccontextmanager + +from typing import cast + +import datetime + +import uvicorn + +from fastapi import BackgroundTasks, FastAPI, HTTPException, status +from fastapi.responses import PlainTextResponse +from fastapi.routing import APIRoute + +from app import __version__, db + +logger = logging.getLogger("multiqc_api") + +logger.info("Starting MultiQC API run logger service") + +# Add timestamp to the uvicorn logger +for h in logging.getLogger("uvicorn.access").handlers: + h.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")) + + +@asynccontextmanager +async def lifespan(_: FastAPI): + yield + + +app = FastAPI( + title="MultiQC API", + description="MultiQC API service, providing run-time information about available " + "" + "" + "" + "" + "" + "" + "" + "" + "updates.", + version=__version__, + license_info={ + "name": "Source code available under the MIT Licence", + "url": "https://github.com/MultiQC/api.multiqc.info/blob/main/LICENSE", + }, + lifespan=lifespan, +) + +db.create_db_and_tables() + + +@app.get("/run") +async def run( + background_tasks: BackgroundTasks, + duration: str, + modules: str, + modules_failed: str, + version_multiqc: str, + version_python: str, + operating_system: str = "", + is_docker: str = "", + is_singularity: str = "", + is_conda: str = "", + is_ci: str = "", +): + """ + Log the modules run by MultiQC + """ + timestamp = datetime.datetime.now().isoformat(timespec="microseconds") + background_tasks.add_task( + db.log_run, + timestamp, + duration=duration, + modules=modules, + modules_failed=modules_failed, + version_multiqc=version_multiqc, + version_python=version_python, + operating_system=operating_system, + is_docker=is_docker, + is_singularity=is_singularity, + is_conda=is_conda, + is_ci=is_ci, + user_ip=None, + ) + + +@app.get("/") +async def index(_: BackgroundTasks): + """ + Root endpoint for the API. + + Returns a list of available endpoints. + """ + routes = [cast(APIRoute, r) for r in app.routes] + return { + "message": "Welcome to the MultiQC runs logging service", + "available_endpoints": [ + {"path": route.path, "name": route.name} for route in routes if route.name != "swagger_ui_redirect" + ], + } + + +@app.get("/health") +async def health(): + """ + Health check endpoint. Checks if the visits table contains records + in the past 15 minutes. + """ + try: + visits = db.get_visit_stats(start=datetime.datetime.now() - datetime.timedelta(minutes=15)) + except Exception as e: + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + if not visits: + raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="No recent visits found") + return PlainTextResponse(content=str(len(visits))) + + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/app/db.py b/app/db.py index 42b6ff2..028b7f2 100644 --- a/app/db.py +++ b/app/db.py @@ -1,4 +1,6 @@ """Functions to interact with the database.""" + +import logging from typing import Optional, Sequence import os @@ -9,10 +11,14 @@ from sqlmodel import create_engine, Field, select, Session, SQLModel +from app.utils import strtobool + sql_url = os.getenv("DATABASE_URL") assert sql_url is not None, sql_url engine = create_engine(sql_url) +logger = logging.getLogger(__name__) + class VisitStats(SQLModel, table=True): """ @@ -37,6 +43,114 @@ class VisitStats(SQLModel, table=True): count: int +class User(SQLModel, table=True): + __tablename__ = "multiqc_api_user" + + id: int = Field(primary_key=True) + ip: str + + +class Run(SQLModel, table=True): + __tablename__ = "multiqc_api_run" + + id: int = Field(primary_key=True) + user_id: Optional[int] = Field(default=None, foreign_key="multiqc_api_user.id") + timestamp: datetime.datetime + duration_seconds: int + version_multiqc: str + version_python: str + operating_system: str + is_docker: bool + is_singularity: bool + is_conda: bool + is_ci: bool + + +class Module(SQLModel, table=True): + __tablename__ = "multiqc_api_module" + + id: int = Field(primary_key=True) + name: str + + +class SuccessRunToModule(SQLModel, table=True): + __tablename__ = "multiqc_api_success_run_to_module" + + id: int = Field(primary_key=True) + run_id: int = Field(foreign_key="multiqc_api_run.id") + module_id: int = Field(foreign_key="multiqc_api_module.id") + + +class FailureRunToModule(SQLModel, table=True): + __tablename__ = "multiqc_api_failure_run_to_module" + + id: int = Field(primary_key=True) + run_id: int = Field(foreign_key="multiqc_api_run.id") + module_id: int = Field(foreign_key="multiqc_api_module.id") + + +def log_run( + timestamp_str: str, + duration: str, + modules: str, + modules_failed: str, + version_multiqc: str, + version_python: str, + operating_system: str, + is_docker: str, + is_singularity: str, + is_conda: str, + is_ci: str, + user_ip: Optional[str] = None, +) -> None: + """Log a run of MultiQC.""" + with Session(engine) as session: + try: + duration_seconds = int(duration) + except ValueError: + logger.warning(f"Could not parse duration: {duration}") + duration_seconds = -1 + + timestamp = datetime.datetime.fromisoformat(timestamp_str) + + run = Run( + timestamp=timestamp, + duration_seconds=duration_seconds, + version_multiqc=version_multiqc, + version_python=version_python, + operating_system=operating_system, + is_docker=strtobool(is_docker), + is_singularity=strtobool(is_singularity), + is_conda=strtobool(is_conda), + is_ci=strtobool(is_ci), + ) + session.add(run) + session.commit() + for _mod in modules.split(","): + module = session.exec(select(Module).where(Module.name == _mod)).first() + if not module: + module = Module(name=_mod) + session.add(module) + session.commit() + session.add(SuccessRunToModule(run_id=run.id, module_id=module.id)) + for _mod in modules_failed.split(","): + module = session.exec(select(Module).where(Module.name == _mod)).first() + if not module: + module = Module(name=_mod) + session.add(module) + session.commit() + session.add(FailureRunToModule(run_id=run.id, module_id=module.id)) + session.add(run) + + if user_ip: + user = User(ip=user_ip) + session.add(user) + session.commit() + run.user_id = user.id + + session.commit() + + class DownloadStats(SQLModel, table=True): """ Daily download statistics. @@ -129,4 +243,3 @@ def insert_download_stats(df: pd.DataFrame) -> pd.DataFrame: session.add(new_entry) session.commit() return df - diff --git a/docker-compose.yml b/docker-compose.yml index bbaf123..c269d08 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,44 @@ version: "3.8" services: - web: - container_name: multiqc_api + visits: + container_name: multiqc_api_visits restart: always build: . - command: uvicorn app.main:app ${UVICORN_RELOAD} --host 0.0.0.0 + command: uvicorn app.app_visits:app ${UVICORN_RELOAD} --host 0.0.0.0 + volumes: + - .:/code + ports: + - "8008:8000" + environment: + # Set in .env + GITHUB_TOKEN: $GITHUB_TOKEN + # Matches the "db" service below + DATABASE_URL: mysql+pymysql://root:1@db:3306/multiqc + depends_on: + wait-for-db: + condition: service_completed_successfully + downloads: + container_name: multiqc_api_downloads + restart: always + build: . + command: uvicorn app.app_downloads:app ${UVICORN_RELOAD} --host 0.0.0.0 + volumes: + - .:/code + ports: + - "8008:8000" + environment: + # Set in .env + GITHUB_TOKEN: $GITHUB_TOKEN + # Matches the "db" service below + DATABASE_URL: mysql+pymysql://root:1@db:3306/multiqc + depends_on: + wait-for-db: + condition: service_completed_successfully + runs: + container_name: multiqc_api_runs + restart: always + build: . + command: uvicorn app.app_runs:app ${UVICORN_RELOAD} --host 0.0.0.0 volumes: - .:/code ports: @@ -14,7 +48,6 @@ services: GITHUB_TOKEN: $GITHUB_TOKEN # Matches the "db" service below DATABASE_URL: mysql+pymysql://root:1@db:3306/multiqc - LOGZIO_TOKEN: $LOGZIO_TOKEN depends_on: wait-for-db: condition: service_completed_successfully