diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml new file mode 100644 index 0000000000..44a88e11d0 --- /dev/null +++ b/.github/workflows/core.yml @@ -0,0 +1,537 @@ +# Main Firedrake CI workflow + +on: + workflow_call: + inputs: + target_branch: + description: The target branch (usually 'master' or 'release') + type: string + required: true + run_tests: + description: Whether to run the test suite + type: boolean + default: true + test_macos: + description: Whether to test using macOS + type: boolean + default: false + deploy_website: + description: Whether to deploy the website + type: boolean + default: false + upload_pypi: + description: Whether to upload an sdist to PyPI + type: boolean + default: false + upload_testpypi: + description: Whether to upload an sdist to TestPyPI + type: boolean + default: false + + workflow_dispatch: + inputs: + target_branch: + description: The target branch (usually 'master' or 'release') + type: string + required: true + run_tests: + description: Whether to run the test suite + type: boolean + default: true + test_macos: + description: Whether to test using macOS + type: boolean + default: false + deploy_website: + description: Whether to deploy the website + type: boolean + default: false + upload_pypi: + description: Whether to upload an sdist to PyPI + type: boolean + default: false + upload_testpypi: + description: Whether to upload an sdist to TestPyPI + type: boolean + default: false + +concurrency: + # Cancel running jobs if new commits are pushed + group: > + ${{ github.workflow }}- + ${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test_linux: + name: Build and test Firedrake (Linux) + strategy: + # We want to know all of the tests which fail, so don't kill real if + # complex fails and vice-versa + fail-fast: false + matrix: + arch: [default, complex] + runs-on: [self-hosted, Linux] + container: + image: ubuntu:latest + env: + OMPI_ALLOW_RUN_AS_ROOT: 1 + OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 + OMP_NUM_THREADS: 1 + OPENBLAS_NUM_THREADS: 1 + FIREDRAKE_CI: 1 + PYOP2_CI_TESTS: 1 + PYOP2_SPMD_STRICT: 1 + EXTRA_PYTEST_ARGS: --splitting-algorithm least_duration --timeout=600 --timeout-method=thread -o faulthandler_timeout=660 + PYTEST_MPI_MAX_NPROCS: 8 + steps: + - name: Fix HOME + # For unknown reasons GitHub actions overwrite HOME to /github/home + # which will break everything unless fixed + # (https://github.com/actions/runner/issues/863) + run: echo "HOME=/root" >> "$GITHUB_ENV" + + - name: Pre-run cleanup + # Make sure the current directory is empty + run: find . -delete + + - uses: actions/checkout@v4 + with: + path: firedrake-repo + + - name: Install system dependencies + run: | + apt-get update + apt-get -y install python3 python3-venv + apt-get -y install \ + $(python3 ./firedrake-repo/scripts/firedrake-configure --arch ${{ matrix.arch }} --show-system-packages) + : # Dependencies needed to run the test suite + apt-get -y install parallel + : # Dependencies needed to build the documentation + apt-get -y install inkscape texlive-full + + # Raise an error if any 'TODO RELEASE' comments remain + - name: Check no 'TODO RELEASE' comments (release only) + if: inputs.target_branch == 'release' + run: | + cd firedrake-repo + if [ -z "$( git grep 'TODO RELEASE' )" ]; then + exit 0 + else + exit 1 + fi + + # TODO: is there a 'blessed commit' or do we just point to 'main'? + # I prefer the latter because then failures are everyone's problem, not just mine! + - name: Install PETSc + run: | + if [ ${{ inputs.target_branch }} = 'release' ]; then + git clone --depth 1 \ + --branch $(python3 ./firedrake-repo/scripts/firedrake-configure --show-petsc-version) \ + https://gitlab.com/petsc/petsc.git + else + git clone --depth 1 https://gitlab.com/petsc/petsc.git + fi + cd petsc + python3 ../firedrake-repo/scripts/firedrake-configure \ + --arch ${{ matrix.arch }} --show-petsc-configure-options | \ + xargs -L1 ./configure --with-make-np=8 --download-slepc + make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-${{ matrix.arch }} + make check + { + echo "PETSC_DIR=/__w/firedrake/firedrake/petsc" + echo "PETSC_ARCH=arch-firedrake-${{ matrix.arch }}" + echo "SLEPC_DIR=/__w/firedrake/firedrake/petsc/arch-firedrake-${{ matrix.arch }}" + } >> "$GITHUB_ENV" + + - name: Install Firedrake + id: install + run: | + export $(python3 ./firedrake-repo/scripts/firedrake-configure --arch "${{ matrix.arch }}" --show-env) + python3 -m venv venv + . venv/bin/activate + + : # Force a rebuild of petsc4py as the cached one will not link to the fresh + : # install of PETSc. + pip cache remove petsc4py + pip cache remove slepc4py + + if [ ${{ inputs.target_branch }} = 'release' ]; then + EXTRA_PIP_FLAGS='' + else + : # Install build dependencies + pip install "$PETSC_DIR"/src/binding/petsc4py + pip install -r ./firedrake-repo/requirements-build.txt + + : # Install runtime dependencies that have been removed from the pyproject.toml + : # because they rely on non-PyPI versions of petsc4py. + pip install --no-build-isolation --no-deps \ + "$PETSC_DIR"/"$PETSC_ARCH"/externalpackages/git.slepc/src/binding/slepc4py + pip install --no-deps ngsPETSc netgen-mesher netgen-occt + + : # We have to pass '--no-build-isolation' to use a custom petsc4py + EXTRA_PIP_FLAGS='--no-build-isolation' + fi + + pip install --verbose $EXTRA_PIP_FLAGS \ + --no-binary h5py \ + --extra-index-url https://download.pytorch.org/whl/cpu \ + './firedrake-repo[ci,docs]' + + firedrake-clean + pip list + + - name: Run firedrake-check + run: | + . venv/bin/activate + firedrake-check + timeout-minutes: 5 + + - name: Run TSFC tests + # Run even if earlier tests failed + if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + run: | + . venv/bin/activate + : # Use pytest-xdist here so we can have a single collated output (not possible + : # for parallel tests) + firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/tsfc + timeout-minutes: 60 + + - name: Run PyOP2 tests + if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + run: | + . venv/bin/activate + : # Use pytest-xdist here so we can have a single collated output (not possible + : # for parallel tests) + firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/pyop2 + firedrake-run-split-tests 2 4 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/pyop2 + firedrake-run-split-tests 3 2 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/pyop2 + firedrake-run-split-tests 4 2 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/pyop2 + timeout-minutes: 15 + + + - name: Run Firedrake tests (nprocs = 1) + if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + run: | + . venv/bin/activate + : # Use pytest-xdist here so we can have a single collated output (not possible + : # for parallel tests) + firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake + timeout-minutes: 60 + + - name: Run tests (nprocs = 2) + if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + run: | + . venv/bin/activate + firedrake-run-split-tests 2 4 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake + timeout-minutes: 30 + + - name: Run tests (nprocs = 3) + if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + run: | + . venv/bin/activate + firedrake-run-split-tests 3 2 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake + timeout-minutes: 60 + + - name: Run tests (nprocs = 4) + if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + run: | + . venv/bin/activate + firedrake-run-split-tests 4 2 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake + timeout-minutes: 15 + + - name: Run tests (nprocs = 5) + if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + run: | + . venv/bin/activate + firedrake-run-split-tests 5 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake + timeout-minutes: 15 + + - name: Run tests (nprocs = 6) + if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + run: | + . venv/bin/activate + firedrake-run-split-tests 6 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake + timeout-minutes: 15 + + - name: Run tests (nprocs = 7) + if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + run: | + . venv/bin/activate + firedrake-run-split-tests 7 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake + timeout-minutes: 15 + + - name: Run tests (nprocs = 8) + if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + run: | + . venv/bin/activate + firedrake-run-split-tests 8 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake + timeout-minutes: 15 + + - name: Run Gusto smoke tests + # Only test Gusto in real mode + if: | + inputs.run_tests && + (success() || steps.install.conclusion == 'success') && + matrix.arch == 'default' + run: | + . venv/bin/activate + git clone --depth 1 https://github.com/firedrakeproject/gusto.git gusto-repo + pip install --verbose ./gusto-repo + python -m pytest -n 8 --verbose \ + gusto-repo/integration-tests/balance/test_saturated_balance.py \ + gusto-repo/integration-tests/equations/test_thermal_sw.py \ + gusto-repo/integration-tests/transport/test_embedded_dg_advection.py + timeout-minutes: 10 + + - name: Run Thetis smoke tests + if: | + inputs.run_tests && + (success() || steps.install.conclusion == 'success') && + matrix.arch == 'default' + run: | + . venv/bin/activate + git clone --depth 1 https://github.com/thetisproject/thetis.git thetis-repo + pip install --verbose ./thetis-repo + python -m pytest -n 8 --verbose thetis-repo/test_adjoint/test_swe_adjoint.py + timeout-minutes: 10 + + - name: Run spyro smoke tests + if: | + inputs.run_tests && + (success() || steps.install.conclusion == 'success') && + matrix.arch == 'default' + run: | + . venv/bin/activate + git clone --depth 1 https://github.com/NDF-Poli-USP/spyro.git spyro-repo + pip install --verbose ./spyro-repo + mpiexec -n 6 python -m pytest spyro-repo/test_integration/ -m parallel[6] + timeout-minutes: 5 + + - name: Run G-ADOPT smoke tests + if: | + inputs.run_tests && + (success() || steps.install.conclusion == 'success') && + matrix.arch == 'default' + run: | + . venv/bin/activate + git clone --depth 1 https://github.com/g-adopt/g-adopt.git g-adopt-repo + pip install --verbose ./g-adopt-repo + make -C g-adopt-repo/demos/mantle_convection/base_case check + timeout-minutes: 5 + + - name: Upload log files + uses: actions/upload-artifact@v4 + if: inputs.run_tests && (success() || steps.install.conclusion == 'success') + with: + name: firedrake-logs-${{ matrix.arch }} + path: pytest_*.log + + - name: Build sdist (release only) + if: | + inputs.target_branch == 'release' && + matrix.arch == 'default' && + (success() || steps.install.conclusion == 'success') + run: | + . venv/bin/activate + pip install build + python -m build ./firedrake-repo --sdist + + - name: Upload sdist (release only) + if: | + inputs.target_branch == 'release' && + matrix.arch == 'default' && + (success() || steps.install.conclusion == 'success') + uses: actions/upload-artifact@v4 + with: + name: dist + path: firedrake-repo/dist/* + + - name: Check bibtex + if: always() && matrix.arch == 'default' + run: | + . venv/bin/activate + make -C firedrake-repo/docs validate-bibtex + + - name: Check documentation links + if: always() && matrix.arch == 'default' + run: | + . venv/bin/activate + make -C firedrake-repo/docs linkcheck + + - name: Build documentation + if: (success() || steps.install.conclusion == 'success') && matrix.arch == 'default' + id: build_docs + run: | + . venv/bin/activate + cd firedrake-repo/docs + make SPHINXOPTS="-t ${{ inputs.target_branch }}" html + make latex + make latexpdf + # : Copy manual to HTML tree + cp build/latex/Firedrake.pdf build/html/_static/manual.pdf + + - name: Upload documentation + uses: actions/upload-pages-artifact@v3 + if: (success() || steps.build_docs.conclusion == 'success') && matrix.arch == 'default' + with: + name: github-pages + path: firedrake-repo/docs/build/html + retention-days: 1 + + - name: Post-run cleanup + if: always() + run: find . -delete + + test_macos: + name: Build and test Firedrake (macOS) + runs-on: [self-hosted, macOS] + if: inputs.test_macos + env: + FIREDRAKE_CI: 1 + OMP_NUM_THREADS: 1 + OPENBLAS_NUM_THREADS: 1 + steps: + - name: Add homebrew to PATH + # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#adding-a-system-path + run: echo "/opt/homebrew/bin" >> "$GITHUB_PATH" + + - name: Pre-run cleanup + # Make sure the current directory is empty + run: find . -delete + + - uses: actions/checkout@v4 + with: + path: firedrake-repo + + - name: Install system dependencies + run: | + brew install $(python3 ./firedrake-repo/scripts/firedrake-configure --arch default --show-system-packages) + + - name: Install PETSc + run: | + if [ ${{ inputs.target_branch }} = 'release' ]; then + git clone --depth 1 \ + --branch $(python3 ./firedrake-repo/scripts/firedrake-configure --show-petsc-version) \ + https://gitlab.com/petsc/petsc.git + else + git clone --depth 1 https://gitlab.com/petsc/petsc.git + fi + cd petsc + python3 ../firedrake-repo/scripts/firedrake-configure \ + --arch default --show-petsc-configure-options | \ + xargs -L1 ./configure --with-make-np=4 + make + make check + { + echo "PETSC_DIR=/Users/github/actions-runner/_work/firedrake/firedrake/petsc" + echo "PETSC_ARCH=arch-firedrake-default" + echo "SLEPC_DIR=/Users/github/actions-runner/_work/firedrake/firedrake/petsc/arch-firedrake-default" + } >> "$GITHUB_ENV" + + - name: Install Firedrake + id: install + run: | + export $(python3 ./firedrake-repo/scripts/firedrake-configure --arch default --show-env) + python3 -m venv venv + . venv/bin/activate + + : # Force a rebuild of petsc4py as the cached one will not link to the fresh + : # install of PETSc. A similar trick may be needed for compiled dependencies + : # like h5py or mpi4py if changing HDF5/MPI libraries. + pip cache remove petsc4py + + if [ ${{ inputs.target_branch }} = 'release' ]; then + EXTRA_PIP_FLAGS='' + else + : # Install build dependencies + pip install "$PETSC_DIR"/src/binding/petsc4py + pip install -r ./firedrake-repo/requirements-build.txt + + : # We have to pass '--no-build-isolation' to use a custom petsc4py + EXTRA_PIP_FLAGS='--no-build-isolation' + fi + + pip install --verbose $EXTRA_PIP_FLAGS \ + --no-binary h5py \ + './firedrake-repo[check]' + + firedrake-clean + pip list + + - name: Run smoke tests + run: | + . venv/bin/activate + firedrake-check + timeout-minutes: 10 + + - name: Post-run cleanup + if: always() + run: | + find . -delete + + lint: + name: Lint codebase + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Setup flake8 annotations + uses: rbialon/flake8-annotations@v1 + - name: Install linting packages + run: pip install flake8 pylint + - name: Lint codebase + run: make lint GITHUB_ACTIONS_FORMATTING=1 + + deploy: + name: Deploy GitHub pages + needs: test_linux + if: inputs.deploy_website + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: http://firedrakeproject.github.io/firedrake/${{ inputs.target_branch }} + runs-on: ubuntu-latest + steps: + - name: Deploy to GitHub Pages + uses: actions/deploy-pages@v4 + + upload_pypi: + name: Upload to PyPI (optional) + if: inputs.upload_pypi && inputs.target_branch == 'release' + needs: test_linux + runs-on: ubuntu-latest + environment: + name: pypi + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist + - name: Push to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + upload_testpypi: + name: Upload to TestPyPI (optional) + if: inputs.upload_testpypi && inputs.target_branch == 'release' + needs: test_linux + runs-on: ubuntu-latest + environment: + name: testpypi + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist + - name: Push to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 45e74f56b9..b26812f672 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,13 +4,7 @@ on: workflow_call: inputs: tag: - description: 'Optional tag (defaults to `latest`)' - required: false - default: 'latest' - type: string - status: - description: 'Firedrake test status' - required: true + description: Docker image tag type: string secrets: # Docker login information @@ -22,8 +16,6 @@ on: jobs: # Firedrake container (just Firedrake) docker_build_vanilla: - # Only run if Firedrake tests pass - if: inputs.status == 'success' strategy: fail-fast: false matrix: @@ -58,8 +50,6 @@ jobs: # # Firedrake container (Firedrake and friends) docker_build_firedrake: - # Only run if "Build Firedrake" succeeds - if: inputs.status == 'success' needs: docker_merge_vanilla uses: ./.github/workflows/docker_build.yml # Only build the 'firedrake' container for 'linux/amd64' because @@ -84,8 +74,6 @@ jobs: # # Firedrake container with TeX docker_build_docdeps: - # Only run if "Build Firedrake" succeeds - if: inputs.status == 'success' needs: docker_merge_vanilla uses: ./.github/workflows/docker_build.yml with: @@ -106,8 +94,6 @@ jobs: # # Firedrake container with Jupyter notebooks docker_build_jupyter: - # Only run if "Build Firedrake" succeeds - if: inputs.status == 'success' needs: docker_merge_firedrake uses: ./.github/workflows/docker_build.yml with: diff --git a/.github/workflows/docker_merge.yml b/.github/workflows/docker_merge.yml index c0b3f2c3a0..b7221fb10c 100644 --- a/.github/workflows/docker_merge.yml +++ b/.github/workflows/docker_merge.yml @@ -7,13 +7,12 @@ on: workflow_call: inputs: target: - description: 'Target docker image name to upload to' + description: Docker image name required: true type: string tag: - description: 'Optional tag (defaults to `latest`)' - required: false - default: 'latest' + description: Docker image tag + required: true type: string secrets: # Docker login information @@ -28,8 +27,7 @@ jobs: steps: - name: Pre-cleanup if: always() - run: | - rm -rf ${{ runner.temp }}/digests + run: rm -rf ${{ runner.temp }}/digests - name: Download digests uses: actions/download-artifact@v4 @@ -47,10 +45,15 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + # NOTE: This action pushes a new image with the given tag but also updates + # the 'latest' tag. - name: Merge and push the per-platform images working-directory: ${{ runner.temp }}/digests run: | - docker buildx imagetools create -t firedrakeproject/${{ inputs.target }}:${{ inputs.tag }} $(printf 'firedrakeproject/${{ inputs.target }}@sha256:%s ' *) + docker buildx imagetools create \ + -t firedrakeproject/${{ inputs.target }}:${{ inputs.tag }} \ + -t firedrakeproject/${{ inputs.target }}:latest \ + $(printf 'firedrakeproject/${{ inputs.target }}@sha256:%s ' *) - name: Inspect image run: | @@ -58,5 +61,4 @@ jobs: - name: Post-cleanup if: always() - run: | - rm -rf ${{ runner.temp }}/digests + run: rm -rf ${{ runner.temp }}/digests diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 0000000000..eea463bacc --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,13 @@ +name: Test pull request + +on: + pull_request: + +jobs: + test: + uses: ./.github/workflows/core.yml + with: + target_branch: ${{ github.base_ref }} + # Only run macOS tests if the PR is labelled 'macOS' + test_macos: ${{ contains(github.event.pull_request.labels.*.name, 'macOS') }} + secrets: inherit diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000000..06b0e316fc --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,16 @@ +name: Deploy after push + +on: + push: + branches: + - master + - release + +jobs: + test: + uses: ./.github/workflows/core.yml + with: + target_branch: ${{ github.ref_name }} + test_macos: true + deploy_website: true + secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..4fa26d2c40 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,23 @@ +name: Publish release + +on: + release: + types: [published] + +jobs: + deploy: + if: ! startsWith( github.ref_name, 'Zenodo' ) + uses: ./.github/workflows/core.yml + with: + target_branch: release + run_tests: false + upload_pypi: true + secrets: inherit + + docker: + if: ! startsWith( github.ref_name, 'Zenodo' ) + name: Build Docker containers + uses: ./.github/workflows/docker.yml + with: + tag: ${{ github.ref }} + secrets: inherit diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 02a258e593..0000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,376 +0,0 @@ -name: CI - -on: - push: - branches: - - master - pull_request: - schedule: - - cron: '0 0 * * 0' - - cron: '0 0 1 * *' # Monthly release - -concurrency: - # Cancel running jobs if new commits are pushed - group: > - ${{ github.workflow }}- - ${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - RELEASE_TAG: latest - -jobs: - test: - name: Install and test Firedrake (Linux) - strategy: - # We want to know all of the tests which fail, so don't kill real if - # complex fails and vice-versa - fail-fast: false - matrix: - arch: [default, complex] - runs-on: [self-hosted, Linux] - container: - image: ubuntu:latest - env: - OMPI_ALLOW_RUN_AS_ROOT: 1 - OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 - OMP_NUM_THREADS: 1 - OPENBLAS_NUM_THREADS: 1 - FIREDRAKE_CI: 1 - PYOP2_CI_TESTS: 1 - PYOP2_SPMD_STRICT: 1 - EXTRA_PYTEST_ARGS: --splitting-algorithm least_duration --timeout=600 --timeout-method=thread -o faulthandler_timeout=660 - PYTEST_MPI_MAX_NPROCS: 8 - steps: - - name: Fix HOME - # For unknown reasons GitHub actions overwrite HOME to /github/home - # which will break everything unless fixed - # (https://github.com/actions/runner/issues/863) - run: echo "HOME=/root" >> "$GITHUB_ENV" - - - name: Pre-run cleanup - # Make sure the current directory is empty - run: find . -delete - - - uses: actions/checkout@v4 - with: - path: firedrake-repo - - - name: Install system dependencies - run: | - apt-get update - apt-get -y install python3 - apt-get -y install \ - $(python3 ./firedrake-repo/scripts/firedrake-configure --arch ${{ matrix.arch }} --show-system-packages) python3-venv parallel - - - name: Install PETSc - run: | - git clone --depth 1 --branch $(python3 ./firedrake-repo/scripts/firedrake-configure --show-petsc-version) https://gitlab.com/petsc/petsc.git - cd petsc - python3 ../firedrake-repo/scripts/firedrake-configure \ - --arch ${{ matrix.arch }} --show-petsc-configure-options | \ - xargs -L1 ./configure --with-make-np=8 --download-slepc - make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-${{ matrix.arch }} - make check - { - echo "PETSC_DIR=/__w/firedrake/firedrake/petsc" - echo "PETSC_ARCH=arch-firedrake-${{ matrix.arch }}" - echo "SLEPC_DIR=/__w/firedrake/firedrake/petsc/arch-firedrake-${{ matrix.arch }}" - } >> "$GITHUB_ENV" - - - name: Install Firedrake - id: install - run: | - export $(python3 ./firedrake-repo/scripts/firedrake-configure --arch ${{ matrix.arch }} --show-env) - python3 -m venv venv - . venv/bin/activate - : # Force a rebuild of petsc4py as the cached one will not link to the fresh - : # install of PETSc. - pip cache remove petsc4py - pip cache remove slepc4py - pip install --verbose \ - --no-binary h5py \ - --extra-index-url https://download.pytorch.org/whl/cpu \ - './firedrake-repo[ci]' - firedrake-clean - pip list - - - name: Run firedrake-check - run: | - . venv/bin/activate - firedrake-check - timeout-minutes: 5 - - - name: Run TSFC tests - # Run even if earlier tests failed - if: success() || steps.install.conclusion == 'success' - run: | - . venv/bin/activate - : # Use pytest-xdist here so we can have a single collated output (not possible - : # for parallel tests) - firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/tsfc - timeout-minutes: 60 - - - name: Run PyOP2 tests - if: success() || steps.install.conclusion == 'success' - run: | - . venv/bin/activate - : # Use pytest-xdist here so we can have a single collated output (not possible - : # for parallel tests) - firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/pyop2 - firedrake-run-split-tests 2 4 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/pyop2 - firedrake-run-split-tests 3 2 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/pyop2 - firedrake-run-split-tests 4 2 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/pyop2 - timeout-minutes: 15 - - - - name: Run Firedrake tests (nprocs = 1) - if: success() || steps.install.conclusion == 'success' - run: | - . venv/bin/activate - : # Use pytest-xdist here so we can have a single collated output (not possible - : # for parallel tests) - firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake - timeout-minutes: 60 - - - name: Run tests (nprocs = 2) - if: success() || steps.install.conclusion == 'success' - run: | - . venv/bin/activate - firedrake-run-split-tests 2 4 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake - timeout-minutes: 30 - - - name: Run tests (nprocs = 3) - if: success() || steps.install.conclusion == 'success' - run: | - . venv/bin/activate - firedrake-run-split-tests 3 2 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake - timeout-minutes: 60 - - - name: Run tests (nprocs = 4) - if: success() || steps.install.conclusion == 'success' - run: | - . venv/bin/activate - firedrake-run-split-tests 4 2 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake - timeout-minutes: 15 - - - name: Run tests (nprocs = 5) - if: success() || steps.install.conclusion == 'success' - run: | - . venv/bin/activate - firedrake-run-split-tests 5 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake - timeout-minutes: 15 - - - name: Run tests (nprocs = 6) - if: success() || steps.install.conclusion == 'success' - run: | - . venv/bin/activate - firedrake-run-split-tests 6 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake - timeout-minutes: 15 - - - name: Run tests (nprocs = 7) - if: success() || steps.install.conclusion == 'success' - run: | - . venv/bin/activate - firedrake-run-split-tests 7 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake - timeout-minutes: 15 - - - name: Run tests (nprocs = 8) - if: success() || steps.install.conclusion == 'success' - run: | - . venv/bin/activate - firedrake-run-split-tests 8 1 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake - timeout-minutes: 15 - - - name: Run Gusto smoke tests - # Only test Gusto in real mode - if: (success() || steps.install.conclusion == 'success') && matrix.arch == 'default' - run: | - . venv/bin/activate - git clone --depth 1 https://github.com/firedrakeproject/gusto.git gusto-repo - pip install --verbose ./gusto-repo - python -m pytest -n 8 --verbose \ - gusto-repo/integration-tests/balance/test_saturated_balance.py \ - gusto-repo/integration-tests/equations/test_thermal_sw.py \ - gusto-repo/integration-tests/transport/test_embedded_dg_advection.py - timeout-minutes: 10 - - - name: Run Thetis smoke tests - if: (success() || steps.install.conclusion == 'success') && matrix.arch == 'default' - run: | - . venv/bin/activate - git clone --depth 1 https://github.com/thetisproject/thetis.git thetis-repo - pip install --verbose ./thetis-repo - python -m pytest -n 8 --verbose thetis-repo/test_adjoint/test_swe_adjoint.py - timeout-minutes: 10 - - - name: Run spyro smoke tests - if: (success() || steps.install.conclusion == 'success') && matrix.arch == 'default' - run: | - . venv/bin/activate - git clone --depth 1 https://github.com/NDF-Poli-USP/spyro.git spyro-repo - pip install --verbose ./spyro-repo - mpiexec -n 6 python -m pytest spyro-repo/test_integration/ -m parallel[6] - timeout-minutes: 5 - - - name: Run G-ADOPT smoke tests - if: (success() || steps.install.conclusion == 'success') && matrix.arch == 'default' - run: | - . venv/bin/activate - git clone --depth 1 https://github.com/g-adopt/g-adopt.git g-adopt-repo - pip install --verbose ./g-adopt-repo - make -C g-adopt-repo/demos/mantle_convection/base_case check - timeout-minutes: 5 - - - name: Upload log files - uses: actions/upload-artifact@v4 - if: success() || steps.install.conclusion == 'success' - with: - name: firedrake-logs-${{ matrix.arch }} - path: pytest_*.log - - - name: Post-run cleanup - if: always() - run: find . -delete - - docker_tag: - name: "Set the Docker release tag" - runs-on: [self-hosted, Linux] - if: github.ref == 'refs/heads/master' - steps: - - name: Set release tag - # Set a release tag if triggered by monthly CRON job - if: github.event.schedule == '0 0 1 * *' - run: | - DATE_TAG="$(date +%Y-%m)" - echo "RELEASE_TAG=$DATE_TAG" >> "$GITHUB_ENV" - - name: Print release tag being used - run: | - echo The release tag is "$RELEASE_TAG" - outputs: - tag: ${{ env.RELEASE_TAG }} - - docker: - name: Build Docker containers - if: github.ref == 'refs/heads/master' - needs: [test, docker_tag] - uses: ./.github/workflows/docker.yml - with: - tag: ${{ needs.docker_tag.outputs.tag }} - status: ${{ needs.test.result }} - secrets: inherit - - lint: - name: Lint codebase - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - name: Setup flake8 annotations - uses: rbialon/flake8-annotations@v1 - - name: Install linting packages - run: pip install flake8 pylint - - name: Lint codebase - run: make lint GITHUB_ACTIONS_FORMATTING=1 - - zenodo_canary: - name: Run zenodo canary - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install deps - run: pip install requests packaging - - name: Zenodo API canary - run: python scripts/firedrake-install --test-doi-resolution - - name: Upload log - uses: actions/upload-artifact@v4 - if: failure() - with: - name: zenodo-canary - path: firedrake-install.log - - build_docs: - name: Build documentation - runs-on: ubuntu-latest - container: - image: firedrakeproject/firedrake-docdeps:latest - outputs: - conclusion: ${{ steps.report.outputs.conclusion }} - steps: - - uses: actions/checkout@v4 - with: - path: firedrake-repo - - - name: Install Firedrake - id: install - run: | - : # Pass '--system-site-packages' so already installed packages can be found - python3 -m venv --system-site-packages venv - . venv/bin/activate - pip uninstall -y firedrake - pip install --verbose './firedrake-repo[docs]' - - - name: Check bibtex - run: | - . venv/bin/activate - make -C firedrake-repo/docs validate-bibtex - - - name: Check documentation links - run: | - . venv/bin/activate - make -C firedrake-repo/docs linkcheck - - - name: Build docs - id: build - if: success() || steps.install.conclusion == 'success' - run: | - . venv/bin/activate - cd firedrake-repo/docs - make html - make latex - make latexpdf - - - name: Copy manual to HTML tree - id: copy - if: success() || steps.build.conclusion == 'success' - run: | - cd firedrake-repo/docs - cp build/latex/Firedrake.pdf build/html/_static/manual.pdf - - - name: Upload artifact - id: upload - if: success() || steps.copy.conclusion == 'success' - uses: actions/upload-pages-artifact@v3 - with: - name: github-pages - path: /__w/firedrake/firedrake/firedrake-repo/docs/build/html - retention-days: 1 - - - name: Report status - id: report - if: success() || steps.upload.conclusion == 'success' - run: echo "conclusion=success" >> "$GITHUB_OUTPUT" - - deploy: - name: Deploy Github pages - needs: build_docs - # Always run this workflow on master, even if linkcheck fails - if: always() && github.ref == 'refs/heads/master' && needs.build_docs.outputs.conclusion == 'success' - permissions: - pages: write - id-token: write - environment: - name: github-pages - url: http://firedrakeproject.github.io/firedrake - runs-on: ubuntu-latest - steps: - - name: Deploy to GitHub Pages - uses: actions/deploy-pages@v4 diff --git a/.github/workflows/test_macos.yml b/.github/workflows/test_macos.yml deleted file mode 100644 index dd987e4a80..0000000000 --- a/.github/workflows/test_macos.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: Install and test Firedrake (macOS) - -on: - push: - branches: - - master - pull_request: - # By default this workflow is run on the "opened", "synchronize" and - # "reopened" events. We add "labelled" so it will run if the PR is given a label. - types: [opened, synchronize, reopened, labeled] - -concurrency: - # Cancels jobs running if new commits are pushed - group: > - ${{ github.workflow }}- - ${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - build: - name: Build Firedrake (macOS) - runs-on: [self-hosted, macOS] - # Only run this action if we are pushing to master or the PR is labelled "macOS" - if: ${{ (github.ref == 'refs/heads/master') || contains(github.event.pull_request.labels.*.name, 'macOS') }} - env: - FIREDRAKE_CI: 1 - OMP_NUM_THREADS: 1 - OPENBLAS_NUM_THREADS: 1 - steps: - - name: Add homebrew to PATH - # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#adding-a-system-path - run: echo "/opt/homebrew/bin" >> "$GITHUB_PATH" - - - name: Pre-run cleanup - # Make sure the current directory is empty - run: find . -delete - - - uses: actions/checkout@v4 - with: - path: firedrake-repo - - - name: Install system dependencies - run: | - brew install $(python3 ./firedrake-repo/scripts/firedrake-configure --arch default --show-system-packages) - - - name: Install PETSc - run: | - git clone --depth 1 --branch $(python3 ./firedrake-repo/scripts/firedrake-configure --show-petsc-version) https://gitlab.com/petsc/petsc.git - cd petsc - python3 ../firedrake-repo/scripts/firedrake-configure \ - --arch default --show-petsc-configure-options | \ - xargs -L1 ./configure --with-make-np=4 - make - make check - - - name: Install Firedrake - id: install - run: | - export $(python3 ./firedrake-repo/scripts/firedrake-configure --arch default --show-env) - python3 -m venv venv - . venv/bin/activate - : # Force a rebuild of petsc4py as the cached one will not link to the fresh - : # install of PETSc. A similar trick may be needed for compiled dependencies - : # like h5py or mpi4py if changing HDF5/MPI libraries. - pip cache remove petsc4py - pip install --verbose --no-binary h5py './firedrake-repo[test]' - firedrake-clean - : # Extra test dependencies - pip install pytest-timeout - pip list - - - name: Run smoke tests - run: | - . venv/bin/activate - firedrake-check - timeout-minutes: 10 - - - name: Post-run cleanup - if: always() - run: | - find . -delete diff --git a/Makefile b/Makefile index 2d5007d4d4..ef491f3efe 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ actionlint: @docker pull rhysd/actionlint:latest @# Exclude SC2046 so it doesn't complain about unquoted $ characters (the @# quoting can prevent proper parsing) - @docker run -e SHELLCHECK_OPTS='--exclude=SC2046' --rm -v $$(pwd):/repo --workdir /repo rhysd/actionlint -color + @docker run -e SHELLCHECK_OPTS='--exclude=SC2046,SC2078' --rm -v $$(pwd):/repo --workdir /repo rhysd/actionlint -color .PHONY: dockerlint dockerlint: diff --git a/docker/Dockerfile.docdeps b/docker/Dockerfile.docdeps deleted file mode 100644 index 832b422b5f..0000000000 --- a/docker/Dockerfile.docdeps +++ /dev/null @@ -1,7 +0,0 @@ -# Dockerfile for Firedrake plus packages needed to build the documentation - -FROM firedrakeproject/firedrake-vanilla-default:latest - -RUN apt-get update \ - && apt-get -y install inkscape texlive-full \ - && rm -rf /var/lib/apt/lists/* diff --git a/docker/Dockerfile.vanilla b/docker/Dockerfile.vanilla index 09d30b6820..3dce406f03 100644 --- a/docker/Dockerfile.vanilla +++ b/docker/Dockerfile.vanilla @@ -26,7 +26,7 @@ ENV PIP_BREAK_SYSTEM_PACKAGES=1 ENV OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 # Download firedrake-configure -RUN curl -O --output-dir /opt https://raw.githubusercontent.com/firedrakeproject/firedrake/master/scripts/firedrake-configure +RUN curl -O --output-dir /opt https://raw.githubusercontent.com/firedrakeproject/firedrake/release/scripts/firedrake-configure # Install system dependencies RUN apt-get update \ @@ -63,4 +63,4 @@ ENV MPICC=$CC # Install Firedrake RUN pip install --verbose --no-binary h5py --src /opt \ - --editable git+https://github.com/firedrakeproject/firedrake.git#egg=firedrake[docker] + --editable git+https://github.com/firedrakeproject/firedrake.git@release#egg=firedrake[docker] diff --git a/docs/source/conf.py b/docs/source/conf.py index 29f5e979a4..695b014a64 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -244,6 +244,13 @@ # Output file base name for HTML help builder. htmlhelp_basename = 'Firedrakedoc' +# Optional header warning about docs version +rst_prolog = """ +.. only:: main + + .. warning:: + You are reading a version of the website built against the unstable ``main`` branch. This content is liable to change without notice and may be inappropriate for your use case. +""" # -- Options for LaTeX output -------------------------------------------- diff --git a/docs/source/documentation.rst b/docs/source/documentation.rst index 2da93622b7..1a90527867 100644 --- a/docs/source/documentation.rst +++ b/docs/source/documentation.rst @@ -6,21 +6,17 @@ .. sidebar:: Current development information. - Firedrake and PyOP2 are continually tested using `GitHub actions `__. + Firedrake is continually tested using `GitHub actions `__. Latest Firedrake status: |firedrakebuild|_ - .. |firedrakebuild| image:: https://github.com/firedrakeproject/firedrake/actions/workflows/build.yml/badge.svg - .. _firedrakebuild: https://github.com/firedrakeproject/firedrake/actions/workflows/build.yml + .. |firedrakebuild| image:: https://github.com/firedrakeproject/firedrake/actions/workflows/push.yml/badge.svg + .. _firedrakebuild: https://github.com/firedrakeproject/firedrake/actions/workflows/push.yml Firedrake and its components are developed on `GitHub - `__ where we also maintain Firedrake-ready - versions of the `FEniCS `__ components - UFL. + `__. * `Firedrake on GitHub `__ - * `TSFC on GitHub `__ * `FIAT on GitHub `__ - * `Firedrake version of UFL on GitHub `__ Getting started =============== diff --git a/docs/source/install.rst b/docs/source/install.rst index 75bbc33f29..d3dc5ca476 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -190,12 +190,6 @@ install Firedrake. To do this perform the following steps: #. Firedrake is now installed and ready for use! -.. warning:: - Until Firedrake has versioned releases (slated for April/May 2025), - :doc:`firedrake-zenodo` will only work with *editable* installations of - Firedrake and its components. To install Firedrake in editable mode you - should follow the instructions :ref:`below`. - .. _firedrake_check: @@ -286,36 +280,6 @@ To resolve the problem you should first remove any existing cached packages:: before re-running the instruction to install Firedrake. -.. _dev_install: - -Developer install ------------------ - -By default Firedrake is installed just like any other Python package into -your environment. If you want to be able to edit Firedrake itself then -an *editable* installation is needed. To install Firedrake in editable -mode you should follow the same -:ref:`steps as for a non-editable install` but replace the -final ``pip install`` command with:: - - $ git clone - $ pip install --no-binary h5py --editable './firedrake[dev]' - -where ```` is ``https://github.com/firedrakeproject/firedrake.git`` -or ``git@github.com:firedrakeproject/firedrake.git`` as preferred. - -The same process applies for Firedrake's dependencies. For example, to install -`FIAT `_ in editable mode you -should run:: - - $ git clone - $ pip install --editable ./fiat - -Note that editable versions of Firedrake's dependencies should be installed *after* -Firedrake is installed. Otherwise installing Firedrake will overwrite -whatever packages you just installed. - - .. _customising: Customising Firedrake @@ -501,6 +465,74 @@ Firedrake can also be used inside the brower using Jupyter notebooks and `Google Colab `_. For more information please see :doc:`here`. +.. _dev_install: + +Developer install +================= + +.. only:: release + + .. warning:: + You are currently looking at the documentation for the current stable + release of Firedrake. For the most recent developer documentation you + should follow the instructions `here `__. + +In order to install a development version of Firedrake the following steps +should be followed: + +#. Install system dependencies :ref:`as before` + +#. Clone and build the *default branch* of PETSc: + + .. code-block:: text + + $ git clone https://gitlab.com/petsc/petsc.git + $ cd petsc + $ python3 ../firedrake-configure --show-petsc-configure-options | xargs -L1 ./configure + $ make PETSC_DIR=/path/to/petsc PETSC_ARCH=arch-firedrake-default all + $ make check + $ cd .. + +#. Clone Firedrake:: + + $ git clone + + where ```` is ``https://github.com/firedrakeproject/firedrake.git`` + or ``git@github.com:firedrakeproject/firedrake.git`` as preferred. + +#. Set the necessary environment variables:: + + $ export $(python3 firedrake-configure --show-env) + +#. Install petsc4py and Firedrake's other build dependencies: + + .. code-block:: text + + $ pip cache remove petsc4py + $ pip install $PETSC_DIR/$PETSC_ARCH/src/binding/petsc4py + $ pip install -r ./firedrake/requirements-dev.txt + +#. Install Firedrake in editable mode without build isolation:: + + $ pip install --no-build-isolation --no-binary h5py --editable './firedrake[check]' + +Editing subpackages +------------------- + +Firedrake dependencies can be cloned and installed in editable mode in an +identical way to Firedrake. For example, to install +`FIAT `_ in editable mode you +should run:: + + $ git clone + $ pip install --editable ./fiat + +For most packages it should not be necessary to pass ``--no-build-isolation``. + +It is important to note that these packages **must be installed after Firedrake**. +This is because otherwise installing Firedrake will overwrite the just-installed +package. + .. _discussion: https://github.com/firedrakeproject/firedrake/discussions .. _issue: https://github.com/firedrakeproject/firedrake/issues .. _homebrew: https://brew.sh/ diff --git a/docs/source/zenodo.rst b/docs/source/zenodo.rst index 4b62b4e305..fcbd08a934 100644 --- a/docs/source/zenodo.rst +++ b/docs/source/zenodo.rst @@ -34,7 +34,7 @@ How to register DOIs for a version of Firedrake This section assumes that you have a Firedrake installation which you have used to conduct some numerical experiment and which you wish to publish or otherwise record for posterity. It is assumed that your -virtualenv is activated or that you otherwise have the firedrake +virtualenv is activated or that you otherwise have the Firedrake scripts in your path. 1. Use ``firedrake-zenodo`` to generate a JSON file containing the @@ -59,16 +59,17 @@ scripts in your path. This will create a file ``firedrake.json`` containing the required information. -2. Create an issue on the Firedrake GitHub page asking that a Zenodo - release be created. Attach the ``firedrake.json`` file to the +2. Create an issue on the `Firedrake GitHub page + `__ asking that a + Zenodo release be created. Attach the ``firedrake.json`` file to the issue. You can create the issue using the correct template `here `__. 3. The Firedrake developers will generate a bespoke Firedrake release containing exactly the set of versions your JSON file specifies, as well as creating a Zenodo record collating these. You will be - provided with a firedrake release tag of the form - ``Firedrake_YYYYMMDD.N``. + provided with a Firedrake release tag of the form + ``Zenodo_YYYYMMDD.N``. You can see an example such a collated record `here `__. @@ -77,7 +78,7 @@ scripts in your path. the DOI) for the collated "meta"-record, which in turn links to all the individual components:: - firedrake-zenodo --bibtex Firedrake_YYYYMMDD.N + firedrake-zenodo --bibtex Zenodo_YYYYMMDD.N Obviously, you substitute in your Firedrake release tag. diff --git a/firedrake/eigensolver.py b/firedrake/eigensolver.py index 4f05d00dbe..9294393a6e 100644 --- a/firedrake/eigensolver.py +++ b/firedrake/eigensolver.py @@ -57,7 +57,7 @@ def __init__(self, A, M=None, bcs=None, bc_shift=0.0, restrict=True): if not SLEPc: raise ImportError( "Unable to import SLEPc, eigenvalue computation not possible " - "(try firedrake-update --slepc)" + "(see https://www.firedrakeproject.org/install.html#slepc)" ) args = A.arguments() diff --git a/firedrake/mg/mesh.py b/firedrake/mg/mesh.py index 42ea985666..446e042b35 100644 --- a/firedrake/mg/mesh.py +++ b/firedrake/mg/mesh.py @@ -122,9 +122,9 @@ def MeshHierarchy(mesh, refinement_levels, try: from ngsPETSc import NetgenHierarchy except ImportError: - raise ImportError("Unable to import netgen and ngsPETSc. Please ensure that netgen and ngsPETSc\ - are installed and available to Firedrake. You can do this via \ - firedrake-update --netgen.") + raise ImportError("Unable to import netgen and ngsPETSc. Please ensure that netgen and ngsPETSc " + "are installed and available to Firedrake (see " + "https://www.firedrakeproject.org/install.html#netgen).") if hasattr(mesh, "netgen_mesh"): return NetgenHierarchy(mesh, refinement_levels, flags=netgen_flags) else: diff --git a/firedrake/mg/opencascade_mh.py b/firedrake/mg/opencascade_mh.py index e320877219..d1e5c6c843 100644 --- a/firedrake/mg/opencascade_mh.py +++ b/firedrake/mg/opencascade_mh.py @@ -20,7 +20,7 @@ def OpenCascadeMeshHierarchy(stepfile, element_size, levels, comm=COMM_WORLD, di from OCC.Core.STEPControl import STEPControl_Reader from OCC.Extend.TopologyUtils import TopologyExplorer except ImportError: - raise ImportError("To use OpenCascadeMeshHierarchy, you must install firedrake with the OpenCascade python bindings (firedrake-update --opencascade).") + raise ImportError("To use OpenCascadeMeshHierarchy, you must install firedrake with the OpenCascade python bindings (https://github.com/tpaviot/pythonocc-core).") if not os.path.isfile(stepfile): raise OSError("%s does not exist" % stepfile) diff --git a/firedrake/scripts/firedrake-zenodo b/firedrake/scripts/firedrake-zenodo index 82475e332b..a5f7a5d1bc 100755 --- a/firedrake/scripts/firedrake-zenodo +++ b/firedrake/scripts/firedrake-zenodo @@ -1,44 +1,43 @@ #! /usr/bin/env python3 + +from __future__ import annotations + +import abc +import contextlib +import dataclasses import hashlib -import re -import importlib import logging -import pathlib import sys import os import subprocess from argparse import ArgumentParser, RawDescriptionHelpFormatter -from collections import OrderedDict import json import time import requests import base64 import datetime +from importlib.metadata import Distribution # Change this to https://sandbox.zenodo.org/api for testing ZENODO_URL = "https://zenodo.org/api" -descriptions = OrderedDict([ - ("firedrake", "an automated finite element system"), - ("ufl", "The Unified Form Language"), - ("fiat", "The Finite Element Automated Tabulator"), - ("petsc", "Portable, Extensible Toolkit for Scientific Computation"), - ("loopy", "Transformation-Based Generation of High-Performance CPU/GPU Code"), - ("slepc", "Scalable Library for Eigenvalue Problem Computations"), -]) - -projects = dict([ - ("firedrake", "firedrakeproject"), - ("ufl", "firedrakeproject"), - ("fiat", "firedrakeproject"), - ("petsc", "firedrakeproject"), - ("loopy", "firedrakeproject"), - ("slepc", "firedrakeproject"), -]) - -components = list(descriptions.keys()) - -optional_components = ("slepc",) +DESCRIPTIONS = { + "firedrake": "Firedrake: an automated finite element system", + "ufl": "UFL: the Unified Form Language", + "fiat": "FIAT: the Finite Element Automated Tabulator", + "petsc": "PETSc: the Portable, Extensible Toolkit for Scientific Computation", + "loopy": "loopy: Transformation-Based Generation of High-Performance CPU/GPU Code", +} + +PYPI_PACKAGE_NAMES = { + "firedrake": "firedrake", + "ufl": "fenics-ufl", + "fiat": "fenics-fiat", + "loopy": "loopy", + "petsc": "petsc4py", +} + +components = list(DESCRIPTIONS.keys()) parser = ArgumentParser(description="""Create Zenodo DOIs for specific versions of Firedrake components. @@ -153,6 +152,43 @@ log = logging.getLogger() cwd = os.getcwd() +class ComponentVersion(abc.ABC): + pass + + +@dataclasses.dataclass(frozen=True) +class ReleaseComponentVersion(ComponentVersion): + version: str + + +@dataclasses.dataclass(frozen=True) +class VCSComponentVersion(ComponentVersion): + commit_id: str + + +def get_component_version(component: str) -> ComponentVersion: + pypi_package_name = PYPI_PACKAGE_NAMES[component] + # This incantation returns a JSON string containing information about where the + # component is installed and whether it is editable or not + # (see https://stackoverflow.com/questions/43348746/how-to-detect-if-module-is-installed-in-editable-mode#75078002). + dist = Distribution.from_name(pypi_package_name) + direct_url_json = dist.read_text("direct_url.json") + + if direct_url_json: + direct_url = json.loads(direct_url_json) + is_editable = direct_url.get("dir_info", {}).get("editable", False) + if is_editable: + # sniff the commit info from the repository + repo = direct_url["url"].removeprefix("file://") + commit_id = get_git_commit_info(repo) + else: + commit_id = direct_url["vcs_info"]["commit_id"] + return VCSComponentVersion(commit_id) + else: + # 'direct_url_json' is 'None' if the package is installed via PyPI + return ReleaseComponentVersion(dist.version) + + def check_call(arguments): if args.log: try: @@ -172,69 +208,12 @@ def check_output(args): raise -class directory(object): - """Context manager that executes body in a given directory""" - def __init__(self, d): - self.d = os.path.abspath(d) - - def __enter__(self): - self.olddir = os.path.abspath(os.getcwd()) - os.chdir(self.d) - - def __exit__(self, *args): - os.chdir(self.olddir) - - -def collect_repo_shas(): - shas = {} - for component in components: - log.info(f"Retrieving git information for {component}") - - repo = None - # handle non-Python components separately - if component in {"petsc", "slepc"}: - repo_root_var = "PETSC_DIR"if component == "petsc" else "SLEPC_DIR" - if repo_root_var in os.environ: - repo = os.environ[repo_root_var] - elif "VIRTUAL_ENV" in os.environ: - venv_path = os.path.join(os.getenv("VIRTUAL_ENV"), "src", component) - if os.path.exists(venv_path): - repo = venv_path - else: - # handle the fact that the fiat Python package is called FIAT - if component == "fiat": - python_package_name = "FIAT" - else: - python_package_name = component - try: - package = importlib.import_module(python_package_name) - repo = pathlib.Path(package.__file__).parent.parent - except ImportError: - pass - - if repo: - try: - shas[component] = get_git_commit_info(component, repo) - except RepositoryNotFoundException: - log.warning("Cannot retrieve git information for component " - f"'{component}' so it will not be included in the " - "release. This may be because the package is not " - "installed in 'editable' mode.") - elif component in optional_components: - log.warning(f"Failed to find optional component '{component}', " - "continuing without it") - else: - log.error(f"Mandatory component '{component}' could not be found.") - sys.exit(1) - return shas - - class RepositoryNotFoundException(RuntimeError): pass -def get_git_commit_info(component, repo): - with directory(repo): +def get_git_commit_info(repo): + with contextlib.chdir(repo): try: check_call(["git", "status"]) except subprocess.CalledProcessError: @@ -243,7 +222,7 @@ def get_git_commit_info(component, repo): try: check_call(["git", "diff-index", "--quiet", "HEAD"]) except subprocess.CalledProcessError: - log.error(f"Component {component} has uncommitted changes, cannot create release") + log.error(f"Repository {repo} contains uncommitted changes, cannot create release") sys.exit(1) return check_output(["git", "rev-parse", "HEAD"]).strip() @@ -271,12 +250,6 @@ def check_github_token_scope(token): log.debug("FIREDRAKE_GITHUB_TOKEN has necessary scopes") -def check_tag(tag): - if re.match(r"Firedrake_20[0-9]{6,6}\.[0-9]+", tag) is None: - log.error("Provided tag '{}' is not a legal Firedrake Zenodo release tag".format(tag)) - sys.exit(1) - - def resolve_tag(tag, components): """Match a tag in component repositories. @@ -295,7 +268,7 @@ def resolve_tag(tag, components): token = os.getenv("FIREDRAKE_GITHUB_TOKEN") if token: check_github_token_scope(token) - gh = github3.login(token=token) + github3.login(token=token) else: log.error("""Need to provide FIREDRAKE_GITHUB_TOKEN for github to resolve tags. @@ -304,21 +277,13 @@ variable FIREDRAKE_GITHUB_TOKEN to a github personal access token.""") sys.exit(1) result = [] for component in components: - repo = gh.repository(projects[component], component) - tags = list(repo.tags()) + repo = get_component_repository(component) try: - found, = [t for t in tags if t.name == tag] + found, = [t for t in repo.tags() if t.name == tag] except ValueError: - if component in optional_components: - continue - if args.skip_missing: - log.warning("Tag '{tag}' does not exist in repository '{repo}'. Continuing".format(tag=tag, repo=repo)) - else: - log.error("Tag '{tag}' does not exist in repository '{repo}'".format( - tag=tag, repo=repo)) - log.error("To continue despite this use --skip-missing") - sys.exit(1) - matching = [t for t in tags if t.commit == found.commit] + log.error(f"Tag '{tag}' does not exist in repository '{repo}'") + sys.exit(1) + matching = [t for t in repo.tags() if t.commit == found.commit] result.append((repo, matching)) return result @@ -428,7 +393,7 @@ def create_description(records, title, doi, additional_dois=None): :returns: A HTML string suitable for zenodo upload.""" links = "\n".join('
  • {name} ({desc}): {doi}
    • '.format( name=repo.name, - desc=descriptions[repo.name], + desc=DESCRIPTIONS[repo.name], doi=record["doi"]) for repo, (_, record) in records) if additional_dois: additional_links = "\n".join('
  • {doi}
    • '.format( @@ -442,15 +407,7 @@ The Firedrake components and dependencies used were: {links}

    - -

    -You can install Firedrake using exactly this set of component versions using: -

    firedrake-install --doi {doi}
    -

    -

    -See firedrakeproject.org/install.html for more information. -

    - """.format(title=title, links=links, doi=doi) + """.format(title=title, links=links) if additional_dois: data = """{data} @@ -458,10 +415,7 @@ See firedrakeproject.org
      {additional_links}
    -

    -

    -You will have to download and install them separately in your Firedrake installation, since we cannot automate it for you. -

    """.format(data=data, additional_links=additional_links) +

    """.format(data=data, additional_links=additional_links) return data @@ -526,16 +480,12 @@ def create_metarecord(tag, title, components, info_file=None, matching_records = match(all_records, possible_tags) if len(matching_records) != len(possible_tags): missing = set(repo for repo, _ in possible_tags).difference(repo for repo, _ in matching_records) - if args.skip_missing: - log.warning("Did not find a Zenodo record for repositories: '{}'. Continuing.".format( - ", ".join(repo.full_name for repo in missing))) - else: - log.error("Did not find a Zenodo record for the following repositories") - for repo in missing: - log.error("{}".format(repo.full_name)) - log.error("") - log.error("If you want to continue anyway use --skip-missing") - sys.exit(1) + log.error("Did not find a Zenodo record for the following repositories") + for repo in missing: + log.error("{}".format(repo.full_name)) + log.error("") + sys.exit(1) + base_url = f"{ZENODO_URL}/deposit/depositions" if os.getenv("FIREDRAKE_ZENODO_TOKEN"): authentication_params = {"access_token": os.getenv("FIREDRAKE_ZENODO_TOKEN")} @@ -580,10 +530,10 @@ with deposit:write scope.""") metadata = { "metadata": { - "title": "Software used in `{}'".format(title), + "title": f"Software used in `{title}'", "upload_type": "software", "creators": [{"name": "firedrake-zenodo"}], - "version": "{}".format(tag), + "version": tag, "access_right": "open", "license": "cc-by", "related_identifiers": ([{"relation": "cites", "identifier": record["doi"]} @@ -599,11 +549,11 @@ with deposit:write scope.""") if depo.status_code >= 400: raise ValueError("Unable to add metadata to deposition {}".format(depo.json())) - components = create_json(matching_records, title, additional_dois=additional_dois) + components_json = create_json(matching_records, title, additional_dois=additional_dois) # This is where files live bucket_url = depo.json()["links"]["bucket"] upload = requests.put("{url}/{filename}".format(url=bucket_url, filename="components.json"), - data=components, + data=components_json, params=authentication_params) if upload.status_code >= 400: raise ValueError("Unable to upload file {}".format(upload.json())) @@ -617,7 +567,7 @@ with deposit:write scope.""") f"Actual: {actual}") checksum(upload.json()["checksum"], - hashlib.md5(components).hexdigest(), + hashlib.md5(components_json).hexdigest(), "components.json") if info_file is not None: @@ -672,7 +622,6 @@ def format_bibtex(record): def create_bibtex(tag): - check_tag(tag) response = requests.get(f"{ZENODO_URL}/records", params={"q": "creators.name:firedrake-zenodo AND version:%s" % tag}) if response.status_code >= 400: @@ -728,25 +677,30 @@ if args.list_meta_records: sys.exit(0) -if args.meta_release: - with open(args.meta_release, "r") as f: +def create_zenodo_meta_release(meta_release: str) -> None: + with open(meta_release, "r") as f: data = json.loads(f.read()) tag = data["tag"] - check_tag(tag) - title = data["title"] - components = data["components"] - info_file = decode_info_file(data["info_file"]) additional_dois = data["additional_dois"] new_version_of = data["new_version_of"] - record = create_metarecord(tag, title, components, info_file=info_file, - additional_dois=additional_dois, - update_record=new_version_of) + record = create_metarecord( + tag, + data["title"], + data["components"], + info_file=decode_info_file(data["info_file"]), + additional_dois=additional_dois, + update_record=new_version_of, + ) record = record.json() log.info("Created Zenodo meta-release.") - log.info("Tag is `{}`".format(tag)) - log.info("DOI is {}".format(record["doi"])) - log.info("Zenodo URL is {}".format(record["links"]["record_html"])) - log.info("BibTeX\n\n```bibtex\n{}\n```".format(format_bibtex(record))) + log.info(f"Tag is `{tag}`") + log.info(f"DOI is {record[doi]}") + log.info(f"Zenodo URL is {record["links"]["record_html"]}") + log.info(f"BibTeX\n\n```bibtex\n{format_bibtex(record)}\n```") + + +if args.meta_release: + create_zenodo_meta_release(args.meta_release) sys.exit(0) @@ -755,8 +709,7 @@ if args.release or not args.input_file: log.error("You must provide a title using the --title option") sys.exit(1) - # Collect hashes from the current repo. - shas = collect_repo_shas() + shas = {c: dataclasses.asdict(get_component_version(c)) for c in components} if args.info_file: shas["metarelease_info_file"] = encode_info_file(args.info_file[0]) else: @@ -783,7 +736,7 @@ if args.additional_dois: for component in components: new_sha = getattr(args, component) if new_sha: - shas[component] = new_sha[0] + shas[component] = {"commit_id": new_sha[0]} if not (args.release or args.input_file): # Dump json and exit. @@ -795,12 +748,13 @@ if not (args.release or args.input_file): try: import github3 - # Shut up the github module - github3.session.__logs__.setLevel(logging.WARNING) except ImportError: log.error("Publishing releases requires the github3 module. Please pip install github3.py") sys.exit(1) +# Shut up the github module +github3.session.__logs__.setLevel(logging.WARNING) + # Github authentication. token = os.getenv("FIREDRAKE_GITHUB_TOKEN") if token: @@ -820,85 +774,126 @@ variable FIREDRAKE_GITHUB_TOKEN to a Github personal access token with public_repo scope.""") sys.exit(1) -fd = gh.repository("firedrakeproject", "firedrake") -tag = time.strftime("Firedrake_%Y%m%d", time.localtime()) -index = -1 +def get_component_repository(component: str) -> github3.Repository: + return gh.repository("firedrakeproject", component) -for r in fd.tags(): - if r.name.startswith(tag): - newindex = int(r.name.split(".")[1]) - index = max(index, newindex) -tag += "." + str(index + 1) -# Verify commits. This ensures that an invalid sha will cause us to fail before we release any component. -# This step also replaces short shas with long ones. This seems to be necessary for release creation. +def generate_unique_release_tag() -> str: + tag = time.strftime("Zenodo_%Y%m%d", time.localtime()) + index = -1 + firedrake_repo = get_component_repository("firedrake") + for release in firedrake_repo.tags(): + if release.name.startswith(tag): + newindex = int(release.name.split(".")[1]) + index = max(index, newindex) + tag += "." + str(index + 1) + return tag -for component in components: - repo = gh.repository(projects[component], component) - try: - commit = repo.commit(shas[component]) - if not commit: - log.error("Failed to find specified commit for %s" % component) +def check_ref_exists(component): + if "version" in shas[component]: + # component is versioned, a GitHub release should already exist + if not release_already_exists(component, shas[component]["version"]): + log.error(f"A release of {component} is referenced but no corresponding release on GitHub can be found, aborting") + sys.exit(1) + else: + repo = get_component_repository(component) + repo.commit(shas[component]["commit_id"]) - shas[component] = commit.sha - except KeyError: - log.warning("No commit specified for %s. No release will be created for this component." % component) - # Also check that the tag name does not already exist. - if any(t.name == tag for t in repo.tags()): - log.warning(f"Repository {component} already has tag {tag}. This should not have happened.") - sys.exit(1) +def make_github_release_or_tag_existing(component: str) -> None: + repo = get_component_repository(component) + + if "version" in shas[component]: + # component is versioned, a GitHub release should already exist + release = get_matching_release(component, shas[component]["version"]) + tag_existing_release = True + commit_id = release.target_commitish + else: + # referencing a specific commit, may need to make a new release -# Now create releases. -message = """This release is specifically created to document the version of -{component} used in a particular set of experiments using -Firedrake. Please do not cite this as a general source for Firedrake -or any of its dependencies. Instead, refer to -https://www.firedrakeproject.org/citing.html""" - - -for component in (set(shas) & set(components)): - log.info("Releasing %s" % component) - repo = gh.repository(projects[component], component) - releases = repo.releases() - just_tag = False - date = datetime.datetime.utcnow().replace(microsecond=0, tzinfo=datetime.timezone.utc).isoformat() - tagger = {"name": "firedrake-zenodo", - "email": "firedrake@imperial.ac.uk", - "date": date} - - for release in releases: - if release.target_commitish == shas[component]: - just_tag = True - break - if just_tag: - repo.create_tag(tag, - message=descriptions[component], - sha=shas[component], + # make sure that 'commit_id' is not truncated + commit_id = repo.commit(shas[component]["commit_id"]) + + if commit_already_exists_in_release(component, commit_id): + tag_existing_release = True + else: + tag_existing_release = False + + if tag_existing_release: + log.info(f"Commit '{commit_id}' found in a pre-existing release for " + f"'{component}', adding tag '{release_tag}' to it") + date = datetime.datetime.utcnow().replace(microsecond=0, tzinfo=datetime.timezone.utc).isoformat() + tagger = {"name": "firedrake-zenodo", + "email": "firedrake@imperial.ac.uk", + "date": date} + repo.create_tag(release_tag, + message=DESCRIPTIONS[component], + sha=commit_id, obj_type="tree", tagger=tagger) else: + log.info(f"Pre-existing release for component '{component}' for commit '{commit_id}' not found, creating a new release '{release_tag}' for it") + body = ( + "This release is specifically created to document the version of " + f"{component} used in a particular set of experiments using " + "Firedrake. Please do not cite this as a general source for Firedrake " + "or any of its dependencies. Instead, refer to " + "https://www.firedrakeproject.org/citing.html" + ) repo.create_release( - tag_name=tag, - target_commitish=shas[component], - name=descriptions[component], - body=message.format(component=component), + tag_name=release_tag, + target_commitish=commit_id, + name=DESCRIPTIONS[component], + body=body, draft=False, prerelease=False) -meta_file = "firedrake-meta-release.json" -with open(meta_file, "w") as f: - data = {"tag": tag, - "title": shas["title"], - "components": sorted(set(shas) & set(components)), - "info_file": shas.get("metarelease_info_file", None), - "new_version_of": shas.get("new_version_of", None), - "additional_dois": shas.get("additional_dois", None)} - f.write(json.dumps(data)) - -log.info("Releases complete. The release tag is %s" % tag) -log.info("Now, you need to create the meta-release") -log.info("Run 'firedrake-zenodo --create-meta-release %s' to do this" % os.path.abspath(meta_file)) -log.info("It is best to wait a short while to ensure that Zenodo is up to date") + +def make_releases() -> None: + for component in components: + check_ref_exists(component) + + for component in components: + make_github_release_or_tag_existing(component) + + meta_file = "firedrake-meta-release.json" + with open(meta_file, "w") as f: + data = {"tag": release_tag, + "title": shas["title"], + "components": components, + "info_file": shas.get("metarelease_info_file", None), + "new_version_of": shas.get("new_version_of", None), + "additional_dois": shas.get("additional_dois", None)} + f.write(json.dumps(data)) + + log.info("Releases complete.") + log.info(f"Now you should create the meta-release with 'firedrake-zenodo --create-meta-release {os.path.abspath(meta_file)}'.") + log.info("It is best to wait a short while to ensure that the new releases are detected by Zenodo.") + + +def release_already_exists(component: str, release_name: str) -> bool: + try: + get_matching_release(component, release_name) + return True + except KeyError: + return False + + +def get_matching_release(component: str, release_name: str) -> github3.Release: + for release in get_component_repository(component).releases(): + if release.tag_name == release_name: + return release + raise KeyError + + +def commit_already_exists_in_release(component: str, commit_id: str) -> bool: + for release in get_component_repository(component).releases(): + if release.target_commitish == commit_id: + return True + return False + + +release_tag = generate_unique_release_tag() +make_releases() diff --git a/pyproject.toml b/pyproject.toml index 55189638ed..cd3e384e58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [project] name = "firedrake" -version = "0.14_dev" +# .. +version = "2025.4.0" description = "An automated system for the portable solution of partial differential equations using the finite element method" readme = "README.rst" license = {file = "LICENSE"} @@ -71,13 +72,6 @@ check = [ "mpi-pytest", "pytest", ] -dev = [ # build dependencies that are needed to run 'make' - "Cython", - "mpi-pytest", - "pybind11", - "pytest", - "setuptools", -] docs = [ "bibtexparser", "matplotlib", # needed to resolve API @@ -163,6 +157,19 @@ script-files = [ [tool.setuptools.package-data] # Unless specified these files will not be installed along with the # rest of the package -firedrake = ["evaluate.h", "locate.c", "icons/*.png"] -firedrake_check = ["Makefile", "tests/firedrake/conftest.py", "tests/*/*/*.py"] -pyop2 = ["*.h", "*.pxd", "*.pyx", "codegen/c/*.c"] +firedrake = [ + "evaluate.h", + "locate.c", + "icons/*.png", +] +firedrake_check = [ + "Makefile", + "tests/firedrake/conftest.py", + "tests/*/*/*.py", +] +pyop2 = [ + "*.h", + "*.pxd", + "*.pyx", + "codegen/c/*.c", +] diff --git a/requirements-build.txt b/requirements-build.txt new file mode 100644 index 0000000000..cff27061d1 --- /dev/null +++ b/requirements-build.txt @@ -0,0 +1,13 @@ +# Core build dependencies (adapted from pyproject.toml) +Cython>=3.0 +libsupermesh +mpi4py>3; python_version >= '3.13' +mpi4py; python_version < '3.13' +numpy +pkgconfig +pybind11 +setuptools>61.2 +rtree>=1.2 + +# Transitive build dependencies +hatchling