Test against PostgreSQL in GitHub Actions

.github/workflows/testing-all-oses.yml

@@ -13,7 +13,7 @@ on:
       - 'GSOC**'
 
 jobs:
-  test:
+  test-sqlite:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -51,3 +51,67 @@ jobs:
       run: micromamba run -n ci env QT_QPA_PLATFORM=offscreen pytest -v -n logical --durations=20 --cov=mslib
         --ignore=tests/_test_msui/test_sideview.py --ignore=tests/_test_msui/test_topview.py --ignore=tests/_test_msui/test_wms_control.py
         tests
+
+  test-pgsql:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"]
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+    - uses: actions/checkout@v4
+    - name: Build requirements.txt file
+      run: |
+        sed -n '/^requirements:/,/^test:/p' localbuild/meta.yaml |
+          sed -e "s/.*- //" |
+          sed -e "s/menuinst.*//" |
+          sed -e "s/.*://" > requirements.tmp.txt
+        cat requirements.d/development.txt >> requirements.tmp.txt
+        echo "pytest-randomly" >> requirements.tmp.txt
+        sed -e '/^$/d' -e '/^#.*$/d' requirements.tmp.txt > requirements.txt
+        rm requirements.tmp.txt
+        cat requirements.txt
+    - name: Get current year and calendar week
+      id: year-and-week
+      run: echo "year-and-week=$(date +%Y-%V)" >> "$GITHUB_OUTPUT"
+    - uses: mamba-org/setup-micromamba@v1
+      with:
+        environment-file: requirements.txt
+        environment-name: ci
+        cache-environment: true
+        # Set the cache key in a way that the cache is invalidated every week on monday
+        cache-environment-key: environment-${{ steps.year-and-week.outputs.year-and-week }}
+    - name: Prepare databases
+      # Create a database for each of the pytest-xdist processes.
+      # The number of created databases might need to be changed in the future, if GitHub runners change.
+      run: PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres
+        -c 'create database mscolab_master;'
+        -c 'create database mscolab_gw0;'
+        -c 'create database mscolab_gw1;'
+        -c 'create database mscolab_gw2;'
+        -c 'create database mscolab_gw3;'
+        -c 'create database mscolab_gw4;'
+        -c 'create database mscolab_gw5;'
+        -c 'create database mscolab_gw6;'
+        -c 'create database mscolab_gw7;'
+    - name: Run tests
+      timeout-minutes: 40
+      # The ignored files can somehow cause the test suite to timeout.
+      # I have no idea yet on why this happens and how to fix it.
+      # Even a module level skip is not enough, they need to be completely ignored.
+      # TODO: fix those tests and drop the ignores
+      run: micromamba run -n ci env QT_QPA_PLATFORM=offscreen pytest -v -n logical --durations=20 --cov=mslib
+        --ignore=tests/_test_msui/test_sideview.py --ignore=tests/_test_msui/test_topview.py --ignore=tests/_test_msui/test_wms_control.py
+        tests

conftest.py

@@ -77,7 +77,7 @@ def keyring_reset():
     keyring.get_keyring().reset()
 
 
-def generate_initial_config():
+def generate_initial_config(worker_id):
     """Generate an initial state for the configuration directory in tests.constants.ROOT_FS
     """
     if not constants.ROOT_FS.exists("msui/testdata"):
@@ -97,7 +97,6 @@ def generate_initial_config():
 
     if not constants.SERVER_CONFIG_FS.exists(constants.MSCOLAB_CONFIG_FILE):
         config_string = f'''
-# SQLALCHEMY_DB_URI = 'mysql://user:[email protected]/mscolab'
 import os
 import logging
 import fs
@@ -152,7 +151,8 @@ def generate_initial_config():
 # enable verification by Mail
 MAIL_ENABLED = False
 
-SQLALCHEMY_DB_URI = 'sqlite:///' + urljoin(DATA_DIR, 'mscolab.db')
+# SQLALCHEMY_DB_URI = 'sqlite:///' + urljoin(DATA_DIR, 'mscolab.db')
+SQLALCHEMY_DB_URI = 'postgresql://postgres:[email protected]/mscolab_{worker_id}'
 
 # enable SQLALCHEMY_ECHO
 SQLALCHEMY_ECHO = True
@@ -212,15 +212,15 @@ def _load_module(module_name, path):
     _load_module("mscolab_settings", path)
 
 
-generate_initial_config()
+generate_initial_config("master")
 
 
 # This import must come after the call to generate_initial_config, otherwise SQLAlchemy will have a wrong database path
 from tests.utils import create_msui_settings_file
 
 
 @pytest.fixture(autouse=True)
-def reset_config():
+def reset_config(worker_id):
     """Reset the configuration directory used in the tests (tests.constants.ROOT_FS) after every test
     """
     # Ideally this would just be constants.ROOT_FS.removetree("/"), but SQLAlchemy complains if the SQLite file is
@@ -230,7 +230,7 @@ def reset_config():
     for e in constants.ROOT_FS.walk.dirs(search="depth"):
         constants.ROOT_FS.removedir(e)
 
-    generate_initial_config()
+    generate_initial_config(worker_id)
     create_msui_settings_file("{}")
     read_config_file()