diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index eddaada7c7..995cbf71e6 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -108,6 +108,18 @@ jobs: # Make sure the current directory is empty run: find . -delete + # Check that the Dockerfile is using the latest Ubuntu version. + # The version is hardcoded into the Dockerfile so that the OS + # for each release is fixed. + - name: Check Dockerfile Ubuntu version + run: | + latest_version=$(grep "VERSION_ID=" /etc/os-release | cut -d '"' -f 2) + docker_version=$(grep FROM docker/Dockerfile.vanilla | cut -d ':' -f 2) + if [[ "$docker_version" != "$latest_version" ]]; then + echo "Ubuntu version ${docker_version} in Dockerfile is out of date with latest version ${latest_version}" + exit 1 + fi + # Use a different mirror to fetch apt packages from to get around # temporary outage. # (https://askubuntu.com/questions/1549622/problem-with-archive-ubuntu-com-most-of-the-servers-are-not-responding) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 934b3e1193..33d2a4dd68 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -6,6 +6,14 @@ on: tag: description: Docker image tag type: string + build_dev: + description: Whether to build a developer vanilla-default container + type: boolean + default: false + branch: + description: Which Firedrake branch to build the containers from + type: string + default: "release" secrets: DOCKERHUB_USER: required: true @@ -16,6 +24,14 @@ on: tag: description: Docker image tag type: string + build_dev: + description: Whether to build a developer vanilla-default container + type: boolean + default: false + branch: + description: Which Firedrake branch to build the containers from + type: string + default: "release" jobs: # Firedrake only @@ -25,11 +41,18 @@ jobs: matrix: os: [Linux, macOS] arch: [default, complex] - include: + platform: [linux/amd64, linux/arm64] + # Cannot use inputs in exclude clauses, so add a dummy matrix dim + build_dev: + - ${{ inputs.build_dev }} + # exclude incompatible os+platform, and only build linux dev containers + exclude: - os: Linux + platform: linux/arm64 + - os: macOS platform: linux/amd64 - os: macOS - platform: linux/arm64 + build_dev: true uses: ./.github/workflows/docker_build.yml with: os: ${{ matrix.os }} @@ -37,6 +60,7 @@ jobs: arch: ${{ matrix.arch }} target: firedrake-vanilla-${{ matrix.arch }} tag: ${{ inputs.tag }} + branch: ${{ inputs.branch }} dockerfile: docker/Dockerfile.vanilla secrets: inherit @@ -50,10 +74,12 @@ jobs: with: target: firedrake-vanilla-${{ matrix.arch }} tag: ${{ inputs.tag }} + tag_latest: ${{ ! inputs.build_dev }} secrets: inherit # Firedrake and friends docker_build_firedrake: + if: ${{ ! inputs.build_dev }} needs: docker_merge_vanilla uses: ./.github/workflows/docker_build.yml # Only build the 'firedrake' container for 'linux/amd64' because @@ -69,15 +95,18 @@ jobs: secrets: inherit docker_merge_firedrake: + if: ${{ ! inputs.build_dev }} uses: ./.github/workflows/docker_merge.yml needs: docker_build_firedrake with: target: firedrake tag: ${{ inputs.tag }} + tag_latest: ${{ ! inputs.build_dev }} secrets: inherit # Firedrake with Jupyter notebooks docker_build_jupyter: + if: ${{ ! inputs.build_dev }} needs: docker_merge_firedrake uses: ./.github/workflows/docker_build.yml with: @@ -89,9 +118,11 @@ jobs: secrets: inherit docker_merge_jupyter: + if: ${{ ! inputs.build_dev }} uses: ./.github/workflows/docker_merge.yml needs: docker_build_jupyter with: target: firedrake-jupyter tag: ${{ inputs.tag }} + tag_latest: ${{ ! inputs.build_dev }} secrets: inherit diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml index ce1be1396e..005237b3a5 100644 --- a/.github/workflows/docker_build.yml +++ b/.github/workflows/docker_build.yml @@ -32,6 +32,12 @@ on: description: 'Firedrake arch to build' required: false type: string + default: 'default' + branch: + description: 'Firedrake branch to build' + required: false + type: string + default: 'release' secrets: # Docker login information DOCKERHUB_USER: @@ -77,6 +83,7 @@ jobs: file: ${{ inputs.dockerfile }} build-args: | ARCH=${{ inputs.arch }} + BRANCH=${{ inputs.branch }} outputs: type=image,name=firedrakeproject/${{ inputs.target }},push-by-digest=true,name-canonical=true,push=true - name: Export digest diff --git a/.github/workflows/docker_merge.yml b/.github/workflows/docker_merge.yml index b7221fb10c..34414a638e 100644 --- a/.github/workflows/docker_merge.yml +++ b/.github/workflows/docker_merge.yml @@ -14,6 +14,10 @@ on: description: Docker image tag required: true type: string + tag_latest: + description: Whether to add additional 'latest' tag to containers + default: false + type: boolean secrets: # Docker login information DOCKERHUB_USER: @@ -50,10 +54,16 @@ jobs: - name: Merge and push the per-platform images working-directory: ${{ runner.temp }}/digests run: | - docker buildx imagetools create \ - -t firedrakeproject/${{ inputs.target }}:${{ inputs.tag }} \ - -t firedrakeproject/${{ inputs.target }}:latest \ - $(printf 'firedrakeproject/${{ inputs.target }}@sha256:%s ' *) + if [[ "${{ inputs.tag_latest }}" == "true" ]]; then + docker buildx imagetools create \ + -t firedrakeproject/${{ inputs.target }}:${{ inputs.tag }} \ + -t firedrakeproject/${{ inputs.target }}:latest \ + $(printf "firedrakeproject/${{ inputs.target }}@sha256:%s " *) + else + docker buildx imagetools create \ + -t firedrakeproject/${{ inputs.target }}:${{ inputs.tag }} \ + $(printf "firedrakeproject/${{ inputs.target }}@sha256:%s " *) + fi - name: Inspect image run: | diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a1430b57e7..9e59221826 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -13,3 +13,12 @@ jobs: # Only run macOS tests if the PR is labelled 'macOS' test_macos: ${{ contains(github.event.pull_request.labels.*.name, 'macOS') }} secrets: inherit + + docker: + name: Build dev Docker containers + uses: ./.github/workflows/docker.yml + with: + tag: dev-main + branch: main + build_dev: true + secrets: inherit diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 45573bd8f5..19d3b73c37 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -15,3 +15,12 @@ jobs: test_macos: true deploy_website: true secrets: inherit + + docker: + name: Build developer Docker containers + uses: ./.github/workflows/docker.yml + with: + tag: dev-${{ github.ref_name }} + branch: ${{ github.ref_name }} + build_dev: true + secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 37a02ef9c3..04dab5b8ee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,4 +19,6 @@ jobs: uses: ./.github/workflows/docker.yml with: tag: ${{ github.ref_name }} + branch: ${{ github.ref_name }} + build_dev: false secrets: inherit diff --git a/docker/Dockerfile.vanilla b/docker/Dockerfile.vanilla index 58c2cb92e1..ff601ed4f5 100644 --- a/docker/Dockerfile.vanilla +++ b/docker/Dockerfile.vanilla @@ -1,10 +1,70 @@ # Dockerfile for a plain Firedrake suitable for testing Firedrake components and applications +# Firedrake is installed as a system package so doesn't require activating a venv. +# +# Three types of build are available: +# 1. A "release" build for a specific version (e.g. 2025.4.2). +# 2. A "release" build with the "release" branch at the latest commit. +# 3. A "developer" build with the "main" branch at the latest commit +# +# A release build will install: +# - Firedrake in editable mode from the specified release tag or branch. +# - PETSc and SLEPc from the corresponding release (specified by firedrake-configure). +# A developer build will install: +# - Firedrake in editable mode from the main branch. +# - PETSc and SLEPc from the main branch. +# - Python dependencies in editable mode that firedrake/main requires the latest main branch for (pyadjoint, ufl, etc). +# +# The two main command line arguments are: +# +# ARCH: +# The `--arch` argument to firedrake-configure, e.g. default, complex +# +# BRANCH: +# Whether to build a release version of Firedrake. +# Options: +# - "release" Build from the latest release branch. +# - Build a specific release (e.g. 2025.4.2). +# - "main" Build a developer install from the latest main branch. +# +# Extra apt packages and PETSc configure arguments can also be +# provided with the following command line arguments: +# +# APT_EXTRA_PACKAGES: +# Extra packages to pass to `apt install` +# +# PETSC_EXTRA_ARGS: +# Extra arguments to pass to PETSc's `configure` script +# +# Example usage for a specific firedrake release with complex scalars: +# $ docker build \ +# --file=Dockerfile.vanilla \ +# --tag=firedrake:2025.4.2-complex \ +# --build-arg ARCH=complex \ +# --build-arg BRANCH=2025.4.2 +# +# Example usage for building from firedrake main with real scalars, ml installed with PETSc, and valgrind installed with apt: +# $ docker build \ +# --file=Dockerfile.vanilla \ +# --tag=firedrake:main-latest \ +# --build-arg ARCH=default \ +# --build-arg BRANCH=main \ +# --build-arg PETSC_EXTRA_ARGS="--download-ml" \ +# --build-arg APT_EXTRA_PACKAGES="valgrind" -FROM ubuntu:latest +FROM ubuntu:24.04 # Firedrake arch to build ARG ARCH="default" +# Is this a release build? +ARG BRANCH="release" + +# Extra system packages +ARG APT_EXTRA_PACKAGES= + +# Extra PETSc configuration options +ARG PETSC_EXTRA_ARGS= + # Set '-o pipefail' to avoid linter error (https://github.com/hadolint/hadolint/wiki/DL4006) SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -18,41 +78,57 @@ ENV TZ=Europe/London # Install 'parallel' because it is needed by 'firedrake-run-split-tests' RUN apt-get update \ - && apt-get -y install curl parallel python3 python3-pip python3-venv sudo \ + && apt-get -y install curl parallel python3 python3-pip python3-venv sudo graphviz graphviz-dev \ && rm -rf /var/lib/apt/lists/* # Allow pip to install into system package locations without prompting ENV PIP_BREAK_SYSTEM_PACKAGES=1 ENV OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 -# Download firedrake-configure -RUN curl -O --output-dir /opt https://raw.githubusercontent.com/firedrakeproject/firedrake/release/scripts/firedrake-configure +# Where to download firedrake-configure from +RUN curl -O --output-dir /opt https://raw.githubusercontent.com/firedrakeproject/firedrake/${BRANCH}/scripts/firedrake-configure # Install system dependencies RUN apt-get update \ && apt-get -y install \ $(python3 /opt/firedrake-configure --arch $ARCH --show-system-packages) \ + ${APT_EXTRA_PACKAGES} \ && rm -rf /var/lib/apt/lists/* # OpenMPI will complain if mpiexec is invoked as root unless these are set ENV OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 -# Install PETSc. We set the compiler optimisation flags manually here to -# remove the default of '-march=native -mtune=native' which is not suitable for Docker images. -RUN git clone --depth 1 --branch $(python3 /opt/firedrake-configure --show-petsc-version) https://gitlab.com/petsc/petsc.git /opt/petsc \ +# Install PETSc. +# 1. Clone the release tag of PETSc if we're installing a Firedrake release. +# 2. We set the compiler optimisation flags manually here to remove the default +# of '-march=native -mtune=native' which is not suitable for Docker images. +# 3. Remove non-essential files to reduce container size (keep SLEPc for slepc4py). +RUN if [ "${BRANCH}" != "main" ]; then \ + CLONE_ARGS="--branch $(python3 /opt/firedrake-configure --show-petsc-version)"; \ + fi; \ + git clone --depth 1 ${CLONE_ARGS} https://gitlab.com/petsc/petsc.git /opt/petsc \ && cd /opt/petsc \ && python3 /opt/firedrake-configure --arch $ARCH --show-petsc-configure-options | \ sed "s/-march=native -mtune=native/-mtune=generic/g" | \ - xargs -L1 ./configure --with-make-np=8 --download-slepc \ - && make \ - && make check \ - && rm -rf ./**/externalpackages \ - && rm -rf ./src/docs \ + xargs -L1 ./configure --with-make-np=8 --download-slepc ${PETSC_EXTRA_ARGS} \ + && make; \ + if [ $? -eq 1 ]; then \ + cat configure.log; \ + exit; \ + fi; \ + make check || exit; \ + for pkg in $(ls ./arch-firedrake-${ARCH}/externalpackages/); do \ + if [ "${pkg}" != "git.slepc" ]; then \ + rm -rf ./arch-firedrake-${ARCH}/${pkg} || exit; \ + fi; \ + done; \ + rm -rf ./src/docs \ && rm -f ./src/**/tutorials/output/* \ && rm -f ./src/**/tests/output/* \ && cd - || exit ENV PETSC_DIR=/opt/petsc PETSC_ARCH=arch-firedrake-$ARCH +ENV SLEPC_DIR=$PETSC_DIR/$PETSC_ARCH ENV PATH="$PETSC_DIR/$PETSC_ARCH/bin:$PATH" ENV HDF5_MPI=ON @@ -61,5 +137,34 @@ ENV CFLAGS="-mtune=generic" CPPFLAGS="-mtune=generic" ENV MPICC=$CC # Install Firedrake -RUN pip install --verbose --no-binary h5py --src /opt \ - --editable git+https://github.com/firedrakeproject/firedrake.git@release#egg=firedrake[docker] +# - petsc4py and slepc4py are installed from source in PETSc repo. +# - slepc4py is installed without build isolation so it links against +# the correct petsc4py version. +# - Firedrake main branch requires main/master branch of some upstream +# packages. These are installed in editable mode. +# The order these are installed is important, e.g. FIAT must be installed +# before UFL otherwise `pip install fiat` will reinstall pypi ufl. +# && cd /opt/firedrake && git checkout JHopeCollins/more-flexible-docker && cd - \ +RUN git clone --branch ${BRANCH} \ + https://github.com/firedrakeproject/firedrake.git /opt/firedrake \ + && pip cache purge \ + && pip install --verbose ${PETSC_DIR}/src/binding/petsc4py \ + && pip install --verbose -r /opt/firedrake/requirements-build.txt \ + && pip install --verbose --no-build-isolation \ + ${SLEPC_DIR}/externalpackages/git.slepc/src/binding/slepc4py \ + && pip install --verbose --no-binary h5py --no-build-isolation \ + --editable '/opt/firedrake[docker]' || exit; \ + if [ ${BRANCH} == "main" ]; then \ + for pkg in \ + "git+https://github.com/dolfin-adjoint/pyadjoint.git#egg=pyadjoint-ad" \ + "git+https://github.com/firedrakeproject/fiat.git#egg=firedrake-fiat" \ + "git+https://github.com/FEniCS/ufl.git#egg=fenics-ufl" \ + ; do \ + pip install --verbose --src /opt --editable ${pkg} || exit; \ + done; \ + fi + +# Run the smoke tests. +RUN cd /opt/firedrake/ \ + && firedrake-check \ + && firedrake-clean diff --git a/docs/source/install.rst b/docs/source/install.rst index b37ef042a0..8513d01801 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -595,7 +595,6 @@ should be followed: $ pip install --no-build-isolation --no-binary h5py --editable './firedrake[check]' - Editing subpackages -------------------