fix: paving the way for postgres support, fix actions and security updates

.github/workflows/data_update.yaml

@@ -44,7 +44,7 @@ jobs:
           version: "0.10.10"
 
       - name: Installing packages
-        run: uv sync --group=scripts
+        run: uv sync --group=scripts --group=app
 
       - name: Running discovery
         run: make discover

.github/workflows/generate_listing.yaml

@@ -41,7 +41,7 @@ jobs:
           version: "0.10.10"
 
       - name: Installing packages
-        run: uv sync --group=scripts
+        run: uv sync --group=scripts --group=app
 
       - name: Running discovery
         run: make discover

.github/workflows/pull_request.yaml

@@ -111,3 +111,4 @@ jobs:
         env:
           TYPESENSE_API_KEY: DUMMY_TYPESENSE_API_KEY
           TYPESENSE_HOST: http://localhost:8108
+          DATABASE_HOST: test_db.sqlite

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "oss4climate"
-version = "0.7.2"
+version = "0.7.3"
 readme = "README.md"
 description = "Package to create listings of Open Source for Energy"
 authors = [
@@ -23,10 +23,13 @@ dependencies = [
     "tqdm>=4.67.1",
     "typesense>=2.0.0",
     "pyyaml>=6.0.3",
+    "httpx>=0.28.1",
     # Security forces below (not actual direct dependencies)
     "cryptography>=46.0.7",
-    "authlib>=1.6.11",
+    "authlib>=1.7.1",
     "python-multipart>=0.0.27",
+    "psycopg2-binary>=2.9.12",
+    "urllib3>=2.7.0",
 ]
 packages = [
     "src/oss4climate",
@@ -47,7 +50,6 @@ scripts = [
 ]
 test = [
     "pytest>=9.0.3",
-    "httpx>=0.28.1",
     "coverage>=7.11.3",
     "pytest-asyncio>=1.3.0",
     # Security only:

src/oss4climate/src/config.py

@@ -1,3 +1,4 @@
+import os
 from pathlib import Path
 from typing import Optional
 from urllib.parse import urlsplit
@@ -12,9 +13,14 @@ class Settings(pydantic_settings.BaseSettings):
     GITLAB_ACCESS_TOKEN: Optional[str] = None
     LOCAL_FOLDER: str = str(Path(__file__).parent.parent.parent.parent / ".data")
     SCRAPING_SQLITE_DB: str = "db.sqlite"
-    # For usage analytics
-    UMAMI_SITE_ID: str = ""
-    APP_SQLITE_DB: str = "app_db.sqlite"
+
+    # Database
+    DATABASE_USERNAME: str | None = None
+    DATABASE_PASSWORD: str | None = None
+    DATABASE_HOST: str
+    DATABASE_NAME: str | None = None
+    DATABASE_PORT: int | None = None
+
     # Identifiants of FTP for export (scripts only)
     EXPORT_FTP_URL: Optional[str] = None
     EXPORT_FTP_USER: Optional[str] = None
@@ -31,6 +37,9 @@ class Settings(pydantic_settings.BaseSettings):
     TYPESENSE_HOST: str = ""
     TYPESENSE_CONNECTION_TIMEOUT: int = 2
 
+    # For data and usage analytics
+    UMAMI_SITE_ID: str | None = None
+
     # Search parameters
     ENABLE_HYBRID_SEARCH: bool = False
 
@@ -83,16 +92,30 @@ def path_scraping_sqlite_db(self) -> str:
         return f"{self.LOCAL_FOLDER}/{self.SCRAPING_SQLITE_DB}"
 
     @property
-    def path_app_sqlite_db(self) -> str:
+    def database_connection_string(self) -> str:
         """
-        Get the full path to the application SQLite database
+        Get the full database connection string
 
-        :return: Full path string to the application database
+        :return: Full database connection string
         """
-        if self.APP_SQLITE_DB.startswith(self.LOCAL_FOLDER):
-            # for backwards compatibility
-            return self.APP_SQLITE_DB
-        return f"{self.LOCAL_FOLDER}/{self.APP_SQLITE_DB}"
+
+        if None not in {
+            self.DATABASE_USERNAME,
+            self.DATABASE_PASSWORD,
+            self.DATABASE_PORT,
+            self.DATABASE_NAME,
+        }:
+            # Postgres case
+            out = f"postgresql+psycopg2://{self.DATABASE_USERNAME}:{self.DATABASE_PASSWORD}@{self.DATABASE_HOST}:{self.DATABASE_PORT}/{self.DATABASE_NAME}"
+        else:
+            # Sqlite case
+            out = f"{self.LOCAL_FOLDER}/{self.DATABASE_HOST}"
+            if not out.endswith(".sqlite"):
+                raise ValueError("Env DATABASE_HOST should point to a SQLite database")
+            db_folder, __ = os.path.split(out)
+            os.makedirs(db_folder, exist_ok=True)
+            out = "sqlite:///" + out
+        return out
 
 
 # Loading settings

src/oss4climate_app/src/database/__init__.py

@@ -40,10 +40,8 @@ class SearchLog(SQLModel, table=True):
 
 
 def _open_engine_and_create_database_if_missing():
-    db_folder, __ = os.path.split(SETTINGS.path_app_sqlite_db)
-    os.makedirs(db_folder, exist_ok=True)
     x = create_engine(
-        f"sqlite:///{SETTINGS.path_app_sqlite_db}",
+        SETTINGS.database_connection_string,
         echo=False,
     )
     # TODO : this currently also creates empty tables for the "oss4climate" part of the code,

src/oss4climate_app/src/routers/ui.py

@@ -38,7 +38,7 @@ def _render_ui_template(
     else:
         canonical_url = f"{url.scheme}://{url.netloc}{url.path}"
     resp = {
-        "UMAMI_SITE_ID": SETTINGS.UMAMI_SITE_ID,
+        "UMAMI_SITE_ID": SETTINGS.UMAMI_SITE_ID if SETTINGS.UMAMI_SITE_ID else "",
         "URL_CODE_REPOSITORY": URL_CODE_REPOSITORY,
         "URL_FEEDBACK_FORM": URL_FEEDBACK_FORM,
         "URL_BASE": SETTINGS.full_url_base,