High Availability Tests #24201
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: High Availability Tests | |
| on: | |
| workflow_dispatch: | |
| inputs: {} | |
| workflow_run: | |
| workflows: ["Tests"] | |
| types: | |
| - completed | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| if: github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: false | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 50 | |
| submodules: true | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| id: setup-python | |
| with: | |
| python-version: '3.12.2' | |
| cache: 'pip' | |
| cache-dependency-path: | | |
| pyproject.toml | |
| # The below is technically a lie as we are technically not | |
| # inside a virtual env, but there is really no reason to bother | |
| # actually creating and activating one as below works just fine. | |
| - name: Export $VIRTUAL_ENV | |
| run: | | |
| venv="$(python -c 'import sys; sys.stdout.write(sys.prefix)')" | |
| echo "VIRTUAL_ENV=${venv}" >> $GITHUB_ENV | |
| - name: Set up uv cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/uv | |
| key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }} | |
| - name: Cached requirements.txt | |
| uses: actions/cache@v4 | |
| id: requirements-cache | |
| with: | |
| path: requirements.txt | |
| key: edb-requirements-${{ hashFiles('pyproject.toml') }} | |
| - name: Compute requirements.txt | |
| if: steps.requirements-cache.outputs.cache-hit != 'true' | |
| run: | | |
| python -m pip install pip-tools | |
| pip-compile --no-strip-extras --all-build-deps \ | |
| --extra test,language-server \ | |
| --output-file requirements.txt pyproject.toml | |
| - name: Install Python dependencies | |
| run: | | |
| python -c "import sys; print(sys.prefix)" | |
| python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt | |
| # Our HA tests currently only work on Postgres 14 (see #6332), | |
| # so check it out before we compute our build cache keys. | |
| - name: Switch back to Postgres 14 | |
| shell: bash | |
| run: | | |
| set -e | |
| cd postgres | |
| # Fetch postgres 14, since the clone was shallow | |
| git fetch origin REL_14_8 --depth=1 | |
| # For whatever reason the tag doesn't get fetched, so find it | |
| # at FETCH_HEAD | |
| git checkout FETCH_HEAD | |
| - name: Compute cache keys | |
| env: | |
| GIST_TOKEN: ${{ secrets.CI_BOT_GIST_TOKEN }} | |
| run: | | |
| mkdir -p shared-artifacts | |
| if [ "$(uname)" = "Darwin" ]; then | |
| find /usr/lib -type f -name 'lib*' -exec stat -f '%N %z' {} + | sort | shasum -a 256 | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt | |
| else | |
| find /usr/lib -type f -name 'lib*' -printf '%P %s\n' | sort | sha256sum | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt | |
| fi | |
| python setup.py -q ci_helper --type cli > shared-artifacts/edgedbcli_git_rev.txt | |
| python setup.py -q ci_helper --type rust >shared-artifacts/rust_cache_key.txt | |
| python setup.py -q ci_helper --type ext >shared-artifacts/ext_cache_key.txt | |
| python setup.py -q ci_helper --type parsers >shared-artifacts/parsers_cache_key.txt | |
| python setup.py -q ci_helper --type postgres >shared-artifacts/postgres_git_rev.txt | |
| python setup.py -q ci_helper --type libpg_query >shared-artifacts/libpg_query_git_rev.txt | |
| echo 'f8cd94309eaccbfba5dea7835b88c78377608a37' >shared-artifacts/stolon_git_rev.txt | |
| python setup.py -q ci_helper --type bootstrap >shared-artifacts/bootstrap_cache_key.txt | |
| echo EDGEDBCLI_GIT_REV=$(cat shared-artifacts/edgedbcli_git_rev.txt) >> $GITHUB_ENV | |
| echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV | |
| echo LIBPG_QUERY_GIT_REV=$(cat shared-artifacts/libpg_query_git_rev.txt) >> $GITHUB_ENV | |
| echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV | |
| echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV | |
| echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV | |
| - name: Upload shared artifacts | |
| uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 | |
| with: | |
| name: shared-artifacts | |
| path: shared-artifacts | |
| retention-days: 1 | |
| # Restore binary cache | |
| - name: Handle cached EdgeDB CLI binaries | |
| uses: actions/cache@v4 | |
| id: cli-cache | |
| with: | |
| path: build/cli | |
| key: edb-cli-v4-${{ env.EDGEDBCLI_GIT_REV }} | |
| - name: Handle cached Rust extensions | |
| uses: actions/cache@v4 | |
| id: rust-cache | |
| with: | |
| path: build/rust_extensions | |
| key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }} | |
| restore-keys: | | |
| edb-rust-v4- | |
| - name: Handle cached Cython extensions | |
| uses: actions/cache@v4 | |
| id: ext-cache | |
| with: | |
| path: build/extensions | |
| key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }} | |
| - name: Handle cached PostgreSQL build | |
| uses: actions/cache@v4 | |
| id: postgres-cache | |
| with: | |
| path: build/postgres/install | |
| key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }} | |
| - name: Handle cached Stolon build | |
| uses: actions/cache@v4 | |
| id: stolon-cache | |
| with: | |
| path: build/stolon/bin | |
| key: edb-stolon-v2-${{ env.STOLON_GIT_REV }} | |
| - name: Handle cached libpg_query build | |
| uses: actions/cache@v4 | |
| id: libpg-query-cache | |
| with: | |
| path: edb/pgsql/parser/libpg_query/libpg_query.a | |
| key: edb-libpg_query-v1-${{ env.LIBPG_QUERY_GIT_REV }} | |
| # Install system dependencies for building | |
| - name: Install system deps | |
| if: | | |
| steps.cli-cache.outputs.cache-hit != 'true' || | |
| steps.rust-cache.outputs.cache-hit != 'true' || | |
| steps.ext-cache.outputs.cache-hit != 'true' || | |
| steps.stolon-cache.outputs.cache-hit != 'true' || | |
| steps.postgres-cache.outputs.cache-hit != 'true' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y uuid-dev libreadline-dev bison flex libprotobuf-c-dev | |
| - name: Install Rust toolchain | |
| if: | | |
| steps.cli-cache.outputs.cache-hit != 'true' || | |
| steps.rust-cache.outputs.cache-hit != 'true' | |
| uses: dsherret/rust-toolchain-file@v1 | |
| # Build EdgeDB CLI | |
| - name: Handle EdgeDB CLI build cache | |
| uses: actions/cache@v4 | |
| if: steps.cli-cache.outputs.cache-hit != 'true' | |
| with: | |
| path: ${{ env.BUILD_TEMP }}/rust/cli | |
| key: edb-cli-build-v7-${{ env.EDGEDBCLI_GIT_REV }} | |
| restore-keys: | | |
| edb-cli-build-v7- | |
| - name: Build EdgeDB CLI | |
| env: | |
| CARGO_HOME: ${{ env.BUILD_TEMP }}/rust/cli/cargo_home | |
| CACHE_HIT: ${{ steps.cli-cache.outputs.cache-hit }} | |
| run: | | |
| if [[ "$CACHE_HIT" == "true" ]]; then | |
| cp -v build/cli/bin/edgedb edb/cli/edgedb | |
| else | |
| python setup.py -v build_cli | |
| fi | |
| # Build Rust extensions | |
| - name: Handle Rust extensions build cache | |
| uses: actions/cache@v4 | |
| if: steps.rust-cache.outputs.cache-hit != 'true' | |
| with: | |
| path: ${{ env.BUILD_TEMP }}/rust/extensions | |
| key: edb-rust-build-v1-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }} | |
| restore-keys: | | |
| edb-rust-build-v1- | |
| - name: Build Rust extensions | |
| env: | |
| CARGO_HOME: ${{ env.BUILD_TEMP }}/rust/extensions/cargo_home | |
| CACHE_HIT: ${{ steps.rust-cache.outputs.cache-hit }} | |
| run: | | |
| if [[ "$CACHE_HIT" != "true" ]]; then | |
| rm -rf ${BUILD_LIB} | |
| mkdir -p build/rust_extensions | |
| rsync -av ./build/rust_extensions/ ${BUILD_LIB}/ | |
| python setup.py -v build_rust | |
| rsync -av ${BUILD_LIB}/ build/rust_extensions/ | |
| rm -rf ${BUILD_LIB} | |
| fi | |
| rsync -av ./build/rust_extensions/edb/ ./edb/ | |
| # Build libpg_query | |
| - name: Build libpg_query | |
| if: | | |
| steps.libpg-query-cache.outputs.cache-hit != 'true' && | |
| steps.ext-cache.outputs.cache-hit != 'true' | |
| run: | | |
| python setup.py build_libpg_query | |
| # Build extensions | |
| - name: Handle Cython extensions build cache | |
| uses: actions/cache@v4 | |
| if: steps.ext-cache.outputs.cache-hit != 'true' | |
| with: | |
| path: ${{ env.BUILD_TEMP }}/edb | |
| key: edb-ext-build-v4-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }} | |
| - name: Build Cython extensions | |
| env: | |
| CACHE_HIT: ${{ steps.ext-cache.outputs.cache-hit }} | |
| BUILD_EXT_MODE: py-only | |
| run: | | |
| if [[ "$CACHE_HIT" != "true" ]]; then | |
| rm -rf ${BUILD_LIB} | |
| mkdir -p ./build/extensions | |
| rsync -av ./build/extensions/ ${BUILD_LIB}/ | |
| BUILD_EXT_MODE=py-only python setup.py -v build_ext | |
| rsync -av ${BUILD_LIB}/ ./build/extensions/ | |
| rm -rf ${BUILD_LIB} | |
| fi | |
| rsync -av ./build/extensions/edb/ ./edb/ | |
| # Build parsers | |
| - name: Handle compiled parsers cache | |
| uses: actions/cache@v4 | |
| id: parsers-cache | |
| with: | |
| path: build/lib | |
| key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }} | |
| restore-keys: | | |
| edb-parsers-v3- | |
| - name: Build parsers | |
| env: | |
| CACHE_HIT: ${{ steps.parsers-cache.outputs.cache-hit }} | |
| run: | | |
| if [[ "$CACHE_HIT" != "true" ]]; then | |
| rm -rf ${BUILD_LIB} | |
| mkdir -p ./build/lib | |
| rsync -av ./build/lib/ ${BUILD_LIB}/ | |
| python setup.py -v build_parsers | |
| rsync -av ${BUILD_LIB}/ ./build/lib/ | |
| rm -rf ${BUILD_LIB} | |
| fi | |
| rsync -av ./build/lib/edb/ ./edb/ | |
| # Build PostgreSQL | |
| - name: Build PostgreSQL | |
| env: | |
| CACHE_HIT: ${{ steps.postgres-cache.outputs.cache-hit }} | |
| run: | | |
| if [[ "$CACHE_HIT" == "true" ]]; then | |
| cp build/postgres/install/stamp build/postgres/ | |
| else | |
| python setup.py build_postgres | |
| cp build/postgres/stamp build/postgres/install/ | |
| fi | |
| # Build Stolon | |
| - name: Set up Go | |
| if: steps.stolon-cache.outputs.cache-hit != 'true' | |
| uses: actions/setup-go@v2 | |
| with: | |
| go-version: 1.16 | |
| - uses: actions/checkout@v4 | |
| if: steps.stolon-cache.outputs.cache-hit != 'true' | |
| with: | |
| repository: edgedb/stolon | |
| path: build/stolon | |
| ref: ${{ env.STOLON_GIT_REV }} | |
| fetch-depth: 0 | |
| submodules: false | |
| - name: Build Stolon | |
| if: steps.stolon-cache.outputs.cache-hit != 'true' | |
| run: | | |
| mkdir -p build/stolon/bin/ | |
| curl -fsSL https://releases.hashicorp.com/consul/1.10.1/consul_1.10.1_linux_amd64.zip | zcat > build/stolon/bin/consul | |
| chmod +x build/stolon/bin/consul | |
| cd build/stolon && make | |
| # Install edgedb-server and populate egg-info | |
| - name: Install edgedb-server | |
| env: | |
| BUILD_EXT_MODE: skip | |
| run: | | |
| # --no-build-isolation because we have explicitly installed all deps | |
| # and don't want them to be reinstalled in an "isolated env". | |
| pip install --no-build-isolation --no-deps -e .[test,docs] | |
| # Refresh the bootstrap cache | |
| - name: Handle bootstrap cache | |
| uses: actions/cache@v4 | |
| id: bootstrap-cache | |
| with: | |
| path: build/cache | |
| key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }} | |
| restore-keys: | | |
| edb-bootstrap-v2- | |
| - name: Bootstrap EdgeDB Server | |
| if: steps.bootstrap-cache.outputs.cache-hit != 'true' | |
| run: | | |
| edb server --bootstrap-only | |
| ha-test: | |
| needs: build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: false | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 50 | |
| submodules: true | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| id: setup-python | |
| with: | |
| python-version: '3.12.2' | |
| cache: 'pip' | |
| cache-dependency-path: | | |
| pyproject.toml | |
| # The below is technically a lie as we are technically not | |
| # inside a virtual env, but there is really no reason to bother | |
| # actually creating and activating one as below works just fine. | |
| - name: Export $VIRTUAL_ENV | |
| run: | | |
| venv="$(python -c 'import sys; sys.stdout.write(sys.prefix)')" | |
| echo "VIRTUAL_ENV=${venv}" >> $GITHUB_ENV | |
| - name: Set up uv cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/uv | |
| key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }} | |
| - name: Download requirements.txt | |
| uses: actions/cache@v4 | |
| with: | |
| path: requirements.txt | |
| key: edb-requirements-${{ hashFiles('pyproject.toml') }} | |
| - name: Install Python dependencies | |
| run: python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt | |
| # Restore the artifacts and environment variables | |
| - name: Download shared artifacts | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| name: shared-artifacts | |
| path: shared-artifacts | |
| - name: Set environment variables | |
| run: | | |
| echo EDGEDBCLI_GIT_REV=$(cat shared-artifacts/edgedbcli_git_rev.txt) >> $GITHUB_ENV | |
| echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV | |
| echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV | |
| echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV | |
| echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV | |
| # Restore build cache | |
| - name: Restore cached EdgeDB CLI binaries | |
| uses: actions/cache@v4 | |
| id: cli-cache | |
| with: | |
| path: build/cli | |
| key: edb-cli-v4-${{ env.EDGEDBCLI_GIT_REV }} | |
| - name: Restore cached Rust extensions | |
| uses: actions/cache@v4 | |
| id: rust-cache | |
| with: | |
| path: build/rust_extensions | |
| key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }} | |
| - name: Restore cached Cython extensions | |
| uses: actions/cache@v4 | |
| id: ext-cache | |
| with: | |
| path: build/extensions | |
| key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }} | |
| - name: Restore compiled parsers cache | |
| uses: actions/cache@v4 | |
| id: parsers-cache | |
| with: | |
| path: build/lib | |
| key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }} | |
| - name: Restore cached PostgreSQL build | |
| uses: actions/cache@v4 | |
| id: postgres-cache | |
| with: | |
| path: build/postgres/install | |
| key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }} | |
| - name: Restore cached Stolon build | |
| uses: actions/cache@v4 | |
| id: stolon-cache | |
| with: | |
| path: build/stolon/bin | |
| key: edb-stolon-v2-${{ env.STOLON_GIT_REV }} | |
| - name: Restore bootstrap cache | |
| uses: actions/cache@v4 | |
| id: bootstrap-cache | |
| with: | |
| path: build/cache | |
| key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }} | |
| - name: Stop if we cannot retrieve the cache | |
| if: | | |
| steps.cli-cache.outputs.cache-hit != 'true' || | |
| steps.rust-cache.outputs.cache-hit != 'true' || | |
| steps.ext-cache.outputs.cache-hit != 'true' || | |
| steps.parsers-cache.outputs.cache-hit != 'true' || | |
| steps.postgres-cache.outputs.cache-hit != 'true' || | |
| steps.stolon-cache.outputs.cache-hit != 'true' || | |
| steps.bootstrap-cache.outputs.cache-hit != 'true' | |
| run: | | |
| echo ::error::Cannot retrieve build cache. | |
| exit 1 | |
| - name: Validate cached binaries | |
| run: | | |
| # Validate EdgeDB CLI | |
| ./build/cli/bin/edgedb --version || exit 1 | |
| # Validate Stolon | |
| ./build/stolon/bin/stolon-sentinel --version || exit 1 | |
| ./build/stolon/bin/stolon-keeper --version || exit 1 | |
| ./build/stolon/bin/stolon-proxy --version || exit 1 | |
| # Validate PostgreSQL | |
| ./build/postgres/install/bin/postgres --version || exit 1 | |
| ./build/postgres/install/bin/pg_config --version || exit 1 | |
| - name: Restore cache into the source tree | |
| run: | | |
| cp -v build/cli/bin/edgedb edb/cli/edgedb | |
| cp -v build/cli/bin/edgedb edb/cli/gel | |
| rsync -av ./build/rust_extensions/edb/ ./edb/ | |
| rsync -av ./build/extensions/edb/ ./edb/ | |
| rsync -av ./build/lib/edb/ ./edb/ | |
| cp build/postgres/install/stamp build/postgres/ | |
| - name: Install edgedb-server | |
| env: | |
| BUILD_EXT_MODE: skip | |
| run: | | |
| # --no-build-isolation because we have explicitly installed all deps | |
| # and don't want them to be reinstalled in an "isolated env". | |
| pip install --no-build-isolation --no-deps -e .[test,docs] | |
| # Run the test | |
| - name: Test | |
| env: | |
| SHARD: ${{ matrix.shard }} | |
| EDGEDB_TEST_HA: 1 | |
| EDGEDB_TEST_CONSUL_PATH: build/stolon/bin/consul | |
| EDGEDB_TEST_STOLON_CTL: build/stolon/bin/stolonctl | |
| EDGEDB_TEST_STOLON_SENTINEL: build/stolon/bin/stolon-sentinel | |
| EDGEDB_TEST_STOLON_KEEPER: build/stolon/bin/stolon-keeper | |
| run: | | |
| edb test -j1 -v -k test_ha_ | |
| workflow-notifications: | |
| if: failure() && github.event_name != 'pull_request' | |
| name: Notify in Slack on failures | |
| needs: | |
| - build | |
| - ha-test | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: 'read' | |
| steps: | |
| - name: Slack Workflow Notification | |
| uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309 | |
| with: | |
| repo_token: ${{secrets.GITHUB_TOKEN}} | |
| slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}} | |
| name: 'Workflow notifications' | |
| icon_emoji: ':hammer:' | |
| include_jobs: 'on-failure' |