diff --git a/.github/workflows/data_update.yaml b/.github/workflows/data_update.yaml index d93b386..0eae45b 100644 --- a/.github/workflows/data_update.yaml +++ b/.github/workflows/data_update.yaml @@ -44,7 +44,7 @@ jobs: version: "0.10.10" - name: Installing packages - run: uv sync --group=scripts + run: uv sync --group=scripts --group=app - name: Running discovery run: make discover diff --git a/.github/workflows/generate_listing.yaml b/.github/workflows/generate_listing.yaml index 87d1e42..fa99deb 100644 --- a/.github/workflows/generate_listing.yaml +++ b/.github/workflows/generate_listing.yaml @@ -41,7 +41,7 @@ jobs: version: "0.10.10" - name: Installing packages - run: uv sync --group=scripts + run: uv sync --group=scripts --group=app - name: Running discovery run: make discover diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index e0ed716..62007be 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -111,3 +111,4 @@ jobs: env: TYPESENSE_API_KEY: DUMMY_TYPESENSE_API_KEY TYPESENSE_HOST: http://localhost:8108 + DATABASE_HOST: test_db.sqlite diff --git a/pyproject.toml b/pyproject.toml index e5bcff7..038a5e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "oss4climate" -version = "0.7.2" +version = "0.7.3" readme = "README.md" description = "Package to create listings of Open Source for Energy" authors = [ @@ -23,10 +23,13 @@ dependencies = [ "tqdm>=4.67.1", "typesense>=2.0.0", "pyyaml>=6.0.3", + "httpx>=0.28.1", # Security forces below (not actual direct dependencies) "cryptography>=46.0.7", - "authlib>=1.6.11", + "authlib>=1.7.1", "python-multipart>=0.0.27", + "psycopg2-binary>=2.9.12", + "urllib3>=2.7.0", ] packages = [ "src/oss4climate", @@ -47,7 +50,6 @@ scripts = [ ] test = [ "pytest>=9.0.3", - "httpx>=0.28.1", "coverage>=7.11.3", "pytest-asyncio>=1.3.0", # Security only: diff --git a/src/oss4climate/src/config.py b/src/oss4climate/src/config.py index 40fd9d0..882b7b9 100644 --- a/src/oss4climate/src/config.py +++ b/src/oss4climate/src/config.py @@ -1,3 +1,4 @@ +import os from pathlib import Path from typing import Optional from urllib.parse import urlsplit @@ -12,9 +13,14 @@ class Settings(pydantic_settings.BaseSettings): GITLAB_ACCESS_TOKEN: Optional[str] = None LOCAL_FOLDER: str = str(Path(__file__).parent.parent.parent.parent / ".data") SCRAPING_SQLITE_DB: str = "db.sqlite" - # For usage analytics - UMAMI_SITE_ID: str = "" - APP_SQLITE_DB: str = "app_db.sqlite" + + # Database + DATABASE_USERNAME: str | None = None + DATABASE_PASSWORD: str | None = None + DATABASE_HOST: str + DATABASE_NAME: str | None = None + DATABASE_PORT: int | None = None + # Identifiants of FTP for export (scripts only) EXPORT_FTP_URL: Optional[str] = None EXPORT_FTP_USER: Optional[str] = None @@ -31,6 +37,9 @@ class Settings(pydantic_settings.BaseSettings): TYPESENSE_HOST: str = "" TYPESENSE_CONNECTION_TIMEOUT: int = 2 + # For data and usage analytics + UMAMI_SITE_ID: str | None = None + # Search parameters ENABLE_HYBRID_SEARCH: bool = False @@ -83,16 +92,30 @@ def path_scraping_sqlite_db(self) -> str: return f"{self.LOCAL_FOLDER}/{self.SCRAPING_SQLITE_DB}" @property - def path_app_sqlite_db(self) -> str: + def database_connection_string(self) -> str: """ - Get the full path to the application SQLite database + Get the full database connection string - :return: Full path string to the application database + :return: Full database connection string """ - if self.APP_SQLITE_DB.startswith(self.LOCAL_FOLDER): - # for backwards compatibility - return self.APP_SQLITE_DB - return f"{self.LOCAL_FOLDER}/{self.APP_SQLITE_DB}" + + if None not in { + self.DATABASE_USERNAME, + self.DATABASE_PASSWORD, + self.DATABASE_PORT, + self.DATABASE_NAME, + }: + # Postgres case + out = f"postgresql+psycopg2://{self.DATABASE_USERNAME}:{self.DATABASE_PASSWORD}@{self.DATABASE_HOST}:{self.DATABASE_PORT}/{self.DATABASE_NAME}" + else: + # Sqlite case + out = f"{self.LOCAL_FOLDER}/{self.DATABASE_HOST}" + if not out.endswith(".sqlite"): + raise ValueError("Env DATABASE_HOST should point to a SQLite database") + db_folder, __ = os.path.split(out) + os.makedirs(db_folder, exist_ok=True) + out = "sqlite:///" + out + return out # Loading settings diff --git a/src/oss4climate_app/src/database/__init__.py b/src/oss4climate_app/src/database/__init__.py index 79d5562..3d834f2 100644 --- a/src/oss4climate_app/src/database/__init__.py +++ b/src/oss4climate_app/src/database/__init__.py @@ -40,10 +40,8 @@ class SearchLog(SQLModel, table=True): def _open_engine_and_create_database_if_missing(): - db_folder, __ = os.path.split(SETTINGS.path_app_sqlite_db) - os.makedirs(db_folder, exist_ok=True) x = create_engine( - f"sqlite:///{SETTINGS.path_app_sqlite_db}", + SETTINGS.database_connection_string, echo=False, ) # TODO : this currently also creates empty tables for the "oss4climate" part of the code, diff --git a/src/oss4climate_app/src/routers/ui.py b/src/oss4climate_app/src/routers/ui.py index feb9e70..a49bd51 100644 --- a/src/oss4climate_app/src/routers/ui.py +++ b/src/oss4climate_app/src/routers/ui.py @@ -38,7 +38,7 @@ def _render_ui_template( else: canonical_url = f"{url.scheme}://{url.netloc}{url.path}" resp = { - "UMAMI_SITE_ID": SETTINGS.UMAMI_SITE_ID, + "UMAMI_SITE_ID": SETTINGS.UMAMI_SITE_ID if SETTINGS.UMAMI_SITE_ID else "", "URL_CODE_REPOSITORY": URL_CODE_REPOSITORY, "URL_FEEDBACK_FORM": URL_FEEDBACK_FORM, "URL_BASE": SETTINGS.full_url_base, diff --git a/uv.lock b/uv.lock index 368c612..efe6930 100644 --- a/uv.lock +++ b/uv.lock @@ -89,15 +89,15 @@ wheels = [ [[package]] name = "authlib" -version = "1.7.0" +version = "1.7.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, { name = "joserfc" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d9/82/4d0603f30c1b4629b1f091bb266b0d7986434891d6940a8c87f8098db24e/authlib-1.7.0.tar.gz", hash = "sha256:b3e326c9aa9cc3ea95fe7d89fd880722d3608da4d00e8a27e061e64b48d801d5", size = 175890, upload-time = "2026-04-18T11:00:28.559Z" } +sdist = { url = "https://files.pythonhosted.org/packages/36/98/7d93f30d029643c0275dbc0bd6d5a6f670661ee6c9a94d93af7ab4887600/authlib-1.7.2.tar.gz", hash = "sha256:2cea25fefcd4e7173bdf1372c0afc265c8034b23a8cd5dcb6a9164b826c64231", size = 176511, upload-time = "2026-05-06T08:10:23.116Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ca/48/c954218b2a250e23f178f10167c4173fecb5a75d2c206f0a67ba58006c26/authlib-1.7.0-py2.py3-none-any.whl", hash = "sha256:e36817afb02f6f0b6bf55f150782499ddd6ddf44b402bb055d3263cc65ac9ae0", size = 258779, upload-time = "2026-04-18T11:00:26.64Z" }, + { url = "https://files.pythonhosted.org/packages/fb/95/adcb68e20c34162e9135f370d6e31737719c2b6f94bc953fe7ed1f10fe21/authlib-1.7.2-py2.py3-none-any.whl", hash = "sha256:3e1faedc9d87e7d56a164eca3ccb6ace0d61b94abe83e92242f8dc8bba9b4a9f", size = 259548, upload-time = "2026-05-06T08:10:21.436Z" }, ] [[package]] @@ -1073,16 +1073,18 @@ wheels = [ [[package]] name = "oss4climate" -version = "0.7.2" +version = "0.7.3" source = { editable = "." } dependencies = [ { name = "authlib" }, { name = "beautifulsoup4" }, { name = "cryptography" }, { name = "docutils" }, + { name = "httpx" }, { name = "jinja2" }, { name = "markdown" }, { name = "pandas" }, + { name = "psycopg2-binary" }, { name = "pyarrow" }, { name = "pydantic" }, { name = "pydantic-settings" }, @@ -1093,6 +1095,7 @@ dependencies = [ { name = "tomlkit" }, { name = "tqdm" }, { name = "typesense" }, + { name = "urllib3" }, ] [package.dev-dependencies] @@ -1114,7 +1117,6 @@ scripts = [ ] test = [ { name = "coverage" }, - { name = "httpx" }, { name = "pygments" }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -1122,13 +1124,15 @@ test = [ [package.metadata] requires-dist = [ - { name = "authlib", specifier = ">=1.6.11" }, + { name = "authlib", specifier = ">=1.7.1" }, { name = "beautifulsoup4", specifier = ">=4.14.2" }, { name = "cryptography", specifier = ">=46.0.7" }, { name = "docutils", specifier = ">=0.22.3" }, + { name = "httpx", specifier = ">=0.28.1" }, { name = "jinja2", specifier = ">=3.1.6" }, { name = "markdown", specifier = ">=3.10" }, { name = "pandas", specifier = ">=3.0,<4" }, + { name = "psycopg2-binary", specifier = ">=2.9.12" }, { name = "pyarrow", specifier = ">=23.0.1" }, { name = "pydantic", specifier = ">=2.12.4" }, { name = "pydantic-settings", specifier = ">=2.7.1" }, @@ -1139,6 +1143,7 @@ requires-dist = [ { name = "tomlkit", specifier = ">=0.13.3" }, { name = "tqdm", specifier = ">=4.67.1" }, { name = "typesense", specifier = ">=2.0.0" }, + { name = "urllib3", specifier = ">=2.7.0" }, ] [package.metadata.requires-dev] @@ -1160,7 +1165,6 @@ scripts = [ ] test = [ { name = "coverage", specifier = ">=7.11.3" }, - { name = "httpx", specifier = ">=0.28.1" }, { name = "pygments", specifier = ">=2.20.0" }, { name = "pytest", specifier = ">=9.0.3" }, { name = "pytest-asyncio", specifier = ">=1.3.0" }, @@ -1259,6 +1263,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, ] +[[package]] +name = "psycopg2-binary" +version = "2.9.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2a/60/a3624f79acea344c16fbef3a94d28b89a8042ddfb8f3e4ca83f538671409/psycopg2_binary-2.9.12.tar.gz", hash = "sha256:5ac9444edc768c02a6b6a591f070b8aae28ff3a99be57560ac996001580f294c", size = 379686, upload-time = "2026-04-21T09:40:34.304Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/9f/ef4ef3c8e15083df90ca35265cfd1a081a2f0cc07bb229c6314c6af817f4/psycopg2_binary-2.9.12-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5cdc05117180c5fa9c40eea8ea559ce64d73824c39d928b7da9fb5f6a9392433", size = 3712459, upload-time = "2026-04-20T23:34:30.549Z" }, + { url = "https://files.pythonhosted.org/packages/b5/01/3dd14e46ba48c1e1a6ec58ee599fa1b5efa00c246d5046cd903d0eeb1af1/psycopg2_binary-2.9.12-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3227a3bc228c10d21011a99245edca923e4e8bf461857e869a507d9a41fe9f6", size = 3822936, upload-time = "2026-04-20T23:34:32.77Z" }, + { url = "https://files.pythonhosted.org/packages/a6/f7/0640e4901119d8a9f7a1784b927f494e2198e213ceb593753d1f2c8b1b30/psycopg2_binary-2.9.12-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:995ce929eede89db6254b50827e2b7fd61e50d11f0b116b29fffe4a2e53c4580", size = 4578676, upload-time = "2026-04-20T23:34:35.18Z" }, + { url = "https://files.pythonhosted.org/packages/b0/55/44df3965b5f297c50cc0b1b594a31c67d6127a9d133045b8a66611b14dfb/psycopg2_binary-2.9.12-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9fe06d93e72f1c048e731a2e3e7854a5bfaa58fc736068df90b352cefe66f03f", size = 4274917, upload-time = "2026-04-20T23:34:37.982Z" }, + { url = "https://files.pythonhosted.org/packages/b0/4b/74535248b1eac0c9336862e8617c765ac94dac76f9e25d7c4a79588c8907/psycopg2_binary-2.9.12-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40e7b28b63aaf737cb3a1edc3a9bbc9a9f4ad3dcb7152e8c1130e4050eddcb7d", size = 5894843, upload-time = "2026-04-20T23:34:40.856Z" }, + { url = "https://files.pythonhosted.org/packages/f2/ba/f1bf8d2ae71868ad800b661099086ee52bc0f8d9f05be1acd8ebb06757cc/psycopg2_binary-2.9.12-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:89d19a9f7899e8eb0656a2b3a08e0da04c720a06db6e0033eab5928aabe60fa9", size = 4110556, upload-time = "2026-04-20T23:34:44.016Z" }, + { url = "https://files.pythonhosted.org/packages/45/46/c15706c338403b7c420bcc0c2905aad116cc064545686d8bf85f1999ea00/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:612b965daee295ae2da8f8218ce1d274645dc76ef3f1abf6a0a94fd57eff876d", size = 3655714, upload-time = "2026-04-20T23:34:46.233Z" }, + { url = "https://files.pythonhosted.org/packages/b3/7c/a2d5dc09b64a4564db242a0fe418fde7d33f6f8259dd2c5b9d7def00fb5a/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:b9a339b79d37c1b45f3235265f07cdeb0cb5ad7acd2ac7720a5920989c17c24e", size = 3301154, upload-time = "2026-04-20T23:34:49.528Z" }, + { url = "https://files.pythonhosted.org/packages/c0/e8/cc8c9a4ce71461f9ec548d38cadc41dc184b34c73e6455450775a9334ccd/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:3471336e1acfd9c7fe507b8bad5af9317b6a89294f9eb37bd9a030bb7bebcdc6", size = 3048882, upload-time = "2026-04-20T23:34:51.86Z" }, + { url = "https://files.pythonhosted.org/packages/19/6a/31e2296bc0787c5ab75d3d118e40b239db8151b5192b90b77c72bc9256e9/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7af18183109e23502c8b2ae7f6926c0882766f35b5175a4cd737ad825e4d7a1b", size = 3351298, upload-time = "2026-04-20T23:34:54.124Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a8/75f4e3e11203b590150abed2cf7794b9c9c9f7eceddae955191138b44dde/psycopg2_binary-2.9.12-cp312-cp312-win_amd64.whl", hash = "sha256:398fcd4db988c7d7d3713e2b8e18939776fd3fb447052daae4f24fa39daede4c", size = 2757230, upload-time = "2026-04-20T23:34:56.242Z" }, + { url = "https://files.pythonhosted.org/packages/91/bb/4608c96f970f6e0c56572e87027ef4404f709382a3503e9934526d7ba051/psycopg2_binary-2.9.12-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7c729a73c7b1b84de3582f73cdd27d905121dc2c531f3d9a3c32a3011033b965", size = 3712419, upload-time = "2026-04-20T23:34:58.754Z" }, + { url = "https://files.pythonhosted.org/packages/5e/af/48f76af9d50d61cf390f8cd657b503168b089e2e9298e48465d029fcc713/psycopg2_binary-2.9.12-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4413d0caef93c5cf50b96863df4c2efe8c269bf2267df353225595e7e15e8df7", size = 3822990, upload-time = "2026-04-20T23:35:00.821Z" }, + { url = "https://files.pythonhosted.org/packages/7a/df/aba0f99397cd811d32e06fc0cc781f1f3ce98bc0e729cb423925085d781a/psycopg2_binary-2.9.12-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4dfcf8e45ebb0c663be34a3442f65e17311f3367089cd4e5e3a3e8e62c978777", size = 4578696, upload-time = "2026-04-20T23:35:03.409Z" }, + { url = "https://files.pythonhosted.org/packages/95/9c/eaa74021ac4e4d5c2f83d82fc6615a63f4fe6c94dc4e94c3990427053f67/psycopg2_binary-2.9.12-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c41321a14dd74aceb6a9a643b9253a334521babfa763fa873e33d89cfa122fb5", size = 4274982, upload-time = "2026-04-20T23:35:05.583Z" }, + { url = "https://files.pythonhosted.org/packages/35/ed/c25deff98bd26187ba48b3b250a3ffc3037c46c5b89362534a15d200e0db/psycopg2_binary-2.9.12-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83946ba43979ebfdc99a3cd0ee775c89f221df026984ba19d46133d8d75d3cd9", size = 5894867, upload-time = "2026-04-20T23:35:07.902Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/8d0e21ca77373c6c9589e5c4528f6e8f0c08c62cafc76fb0bddb7a2cee22/psycopg2_binary-2.9.12-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:411e85815652d13560fbe731878daa5d92378c4995a22302071890ec3397d019", size = 4110578, upload-time = "2026-04-20T23:35:10.149Z" }, + { url = "https://files.pythonhosted.org/packages/00/fc/f481e2435bd8f742d0123309174aae4165160ad3ef17c1b99c3622c241d2/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1c8ad4c08e00f7679559eaed7aff1edfffc60c086b976f93972f686384a95e2c", size = 3655816, upload-time = "2026-04-20T23:35:12.56Z" }, + { url = "https://files.pythonhosted.org/packages/53/79/b9f46466bdbe9f239c96cde8be33c1aace4842f06013b47b730dc9759187/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:00814e40fa23c2b37ef0a1e3c749d89982c73a9cb5046137f0752a22d432e82f", size = 3301307, upload-time = "2026-04-20T23:35:15.029Z" }, + { url = "https://files.pythonhosted.org/packages/3f/19/7dc003b32fe35024df89b658104f7c8538a8b2dcbde7a4e746ce929742e7/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:98062447aebc20ed20add1f547a364fd0ef8933640d5372ff1873f8deb9b61be", size = 3048968, upload-time = "2026-04-20T23:35:16.757Z" }, + { url = "https://files.pythonhosted.org/packages/91/58/2dbd7db5c604d45f4950d988506aae672a14126ec22998ced5021cbb76bb/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:66a7685d7e548f10fb4ce32fb01a7b7f4aa702134de92a292c7bd9e0d3dbd290", size = 3351369, upload-time = "2026-04-20T23:35:18.933Z" }, + { url = "https://files.pythonhosted.org/packages/42/ee/dee8dcaad07f735824de3d6563bc67119fa6c28257b17977a8d624f02fab/psycopg2_binary-2.9.12-cp313-cp313-win_amd64.whl", hash = "sha256:b6937f5fe4e180aeee87de907a2fa982ded6f7f15d7218f78a083e4e1d68f2a0", size = 2757347, upload-time = "2026-04-20T23:35:21.283Z" }, +] + [[package]] name = "py-key-value-aio" version = "0.4.4" @@ -1974,11 +2008,11 @@ wheels = [ [[package]] name = "urllib3" -version = "2.6.3" +version = "2.7.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] [[package]]