test: add basic system-level test suite

.github/actions/deploy/action.yaml

@@ -0,0 +1,64 @@
+name: deploy-suite
+description: |
+  A Github action to build and deploy a suite deployed on an ecflow server.
+inputs:
+  troika_user:
+    description: User used to submit troika job.
+    required: true
+  sbatch_options:
+    description: List of SBATCH directives
+    required: false
+  site:
+    description: HPC site name.
+    required: false
+    default: hpc-batch
+  github_token:
+    description: Github token.
+    required: true
+  ecflow_host:
+    description: ecflow server hostname.
+    required: true
+  ecflow_port:
+    description: ecflow server port.
+    required: true
+  suite_name:
+    description: Name of the suite.
+    required: true
+  build_options:
+    description: List of options to pass to the build script.
+    required: false
+    default: "limit_workers=1 "
+runs:
+  using: composite
+  steps:
+    - name: Build and deploy suite to ecflow server
+      uses: ecmwf-actions/reusable-workflows/ci-hpc-generic@v2
+      with:
+        template: |
+          set -eux
+          echo "Job is running on ${{ runner.os }}"
+
+          # Load modules of interest
+          module load ecflow
+          module load wellies
+
+          # Clone the repository
+          PACKAGE_NAME=anemoi-docs
+          PACKAGE_BRANCH=${{ github.head_ref || github.ref_name }}
+          git clone -b $PACKAGE_BRANCH https://${{ inputs.github_token }}@github.com/ecmwf/${PACKAGE_NAME}.git
+          cd $PACKAGE_NAME
+          cd tests/system-level
+
+          # Set ecflow server variables
+          export ECF_HOST=${{ inputs.ecflow_host }}
+          export ECF_PORT=${{ inputs.ecflow_port }}
+
+          # Build the suite definition file
+          BUILD_DIR=${SCRATCHDIR}/pyflow/${{ inputs.suite_name }}/build
+          ./build.sh -s name=${{ inputs.suite_name }} ${{ inputs.build_options }} -y
+
+          # Deploy the suite. If a suite is already running it will fail. User should deal with its zombies
+          ecflow_client --replace=/anemoi_tests/${{ inputs.suite_name }} $HOME/pyflow/anemoi_tests/${{ inputs.suite_name }}/${{ inputs.suite_name }}.def
+        sbatch_options: ${{ inputs.sbatch_options }}
+        troika_user: ${{ inputs.troika_user }}
+        site: ${{ inputs.site }}

.github/workflows/nightly_ci_hpc.yaml

@@ -0,0 +1,51 @@
+name: nightly-system-level-test
+
+on:
+  schedule:
+    - cron: '0 23 * * 0' # Nightly at 23pm on all main branches
+  # pull_request:
+  #   types: [opened, synchronize, reopened]
+  #   branches:
+  #     - main
+  #     - develop
+
+
+jobs:
+  check-commits:
+    runs-on: ubuntu-latest
+    name: Check latest commits
+    outputs:
+        should_run: ${{ steps.check-commits.outputs.should_run }}
+    steps:
+      - name: Check for recent commits on main in anemoi repos
+        id: check-commits
+        run: |
+          REPOS=("anemoi-core" "anemoi-datasets" "anemoi-docs")
+          SHOULD_RUN=false
+
+          for REPO in "${REPOS[@]}"; do
+            echo "Checking repository: $REPO"
+
+            # Fetch the latest commit date on the main branch from GitHub API
+            LATEST_COMMIT_DATE=$(curl -s "https://api.github.com/repos/ecmwf/$REPO/commits/main" | jq -r '.commit.committer.date')
+
+            if [ "$(date -d "$LATEST_COMMIT_DATE" +%s)" -gt "$(date -d "24 hours ago" +%s)" ]; then
+              echo "Recent commit found in $REPO: $LATEST_COMMIT_DATE"
+              SHOULD_RUN=true
+            fi
+          done
+
+          echo "should_run=$SHOULD_RUN" >> "$GITHUB_OUTPUT"
+
+  system_level_test:
+    needs: check-commits
+    if: needs.check-commits.outputs.should_run == 'true'
+    uses: ./.github/workflows/system_level_test.yaml
+    with:
+      suite_name: "nightly"
+      build_options: ""
+    secrets:
+      gh_token: ${{ secrets.GH_REPO_READ_TOKEN }}
+      troika_user: ${{ secrets.HPC_SYSTEM_TEST_USER }}
+      ecflow_host: ${{ secrets.HPC_SYSTEM_TEST_HOST }}
+      ecflow_port: ${{ secrets.HPC_SYSTEM_TEST_PORT }}

.github/workflows/on_demand_ci_hpc.yaml

@@ -0,0 +1,61 @@
+name: on-demand-system-level-test
+
+on:
+  workflow_dispatch:
+    inputs:
+      anemoi_docs_branch:
+        description: Branch for anemoi-docs to be used to deploy the anemoi_tests suite.
+        required: false
+      anemoi_datasets_branch:
+        description: Branch for anemoi-datasets to be used to build the datasets environment for testing.
+        required: false
+        default: "main"
+      anemoi_training_branch:
+        description: Branch for anemoi-training to be used to build the training environment for testing.
+        required: false
+        default: "main"
+
+
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      suite_name: ${{ steps.normalize.outputs.suite_name }}
+      build_options: ${{ steps.set_build_options.outputs.build_options }}
+    steps:
+      - name: Normalize suite_name
+        id: normalize
+        run: echo "suite_name=${GITHUB_TRIGGERING_ACTOR//-/_}" >> "$GITHUB_OUTPUT"
+        env:
+          GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }}
+      - name: Set build options
+        id: set_build_options
+        run: |
+          build_options=""
+
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            docs_branch="${{ github.event.inputs.anemoi_docs_branch }}"
+            datasets_branch="${{ github.event.inputs.anemoi_datasets_branch }}"
+            training_branch="${{ github.event.inputs.anemoi_training_branch }}"
+
+            # Default to ref name if docs branch not specified
+            if [[ -z "$docs_branch" ]]; then
+              docs_branch="${{ github.ref_name }}"
+            fi
+
+            build_options="anemoi_docs_branch=$docs_branch anemoi_datasets_branch=$datasets_branch anemoi_training_branch=$training_branch"
+          fi
+
+          echo "build_options=$build_options" >> "$GITHUB_OUTPUT"
+
+  system_level_test:
+    needs: prepare
+    uses: ./.github/workflows/system_level_test.yaml
+    with:
+      suite_name: ${{ needs.prepare.outputs.suite_name }}
+      build_options: ${{ needs.prepare.outputs.build_options }}
+    secrets:
+      gh_token: ${{ secrets.GH_REPO_READ_TOKEN }}
+      troika_user: ${{ secrets.HPC_SYSTEM_TEST_USER }}
+      ecflow_host: ${{ secrets.HPC_SYSTEM_TEST_HOST }}
+      ecflow_port: ${{ secrets.HPC_SYSTEM_TEST_PORT }}

.github/workflows/system_level_test.yaml

@@ -0,0 +1,146 @@
+name: system-level-test
+description: |
+  A Github workflow to build, deploy, run and check that a suite
+  finished well on an ecflow server.
+
+on:
+  workflow_call:
+    inputs:
+      build_options:
+        required: true
+        type: string
+      suite_name:
+        required: true
+        type: string
+    secrets:
+      gh_token:
+        required: true
+      troika_user:
+        required: true
+      ecflow_host:
+        required: true
+      ecflow_port:
+        required: true
+
+jobs:
+  deploy:
+    runs-on: hpc
+    name: Deploy
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build and deploy the suite
+        uses: ./.github/actions/deploy
+        with:
+          sbatch_options: |
+            #SBATCH --job-name=anemoi_test_build_deploy
+            #SBATCH --time=00:10:00
+            #SBATCH --qos=nf
+          site: hpc-batch
+          troika_user: ${{ secrets.troika_user }}
+          github_token: ${{ secrets.gh_token }}
+          ecflow_host: ${{ secrets.ecflow_host }}
+          ecflow_port: ${{ secrets.ecflow_port }}
+          suite_name: ${{ inputs.suite_name }}
+          build_options: ${{ inputs.build_options }}
+  run:
+    runs-on: hpc
+    name: Run
+    needs: deploy
+    steps:
+    - uses: ecmwf/reusable-workflows/ci-hpc-generic@v2
+      name: Run suite from ecflow server
+      with:
+        template: |
+          set -eux
+          echo "Job is running on ${{ runner.os }}"
+
+          # Load modules of interest
+          module load ecflow
+
+          # Set ecflow server variables
+          export ECF_HOST=${{ secrets.ecflow_host }}
+          export ECF_PORT=${{ secrets.ecflow_port }}
+
+          # Run the suite
+          ecflow_client --begin=/anemoi_tests
+          ecflow_client --resume=/anemoi_tests/${{ inputs.suite_name }}
+        sbatch_options: |
+            #SBATCH --job-name=anemoi_test_build
+            #SBATCH --time=00:10:00
+            #SBATCH --qos=nf
+        troika_user: ${{ secrets.troika_user }}
+        site: hpc-batch
+  monitor:
+    runs-on: hpc
+    name: Monitor
+    needs: run
+    steps:
+      - name: Check that the suite is complete
+        uses: ecmwf/reusable-workflows/hpc/ecflow/wait-for-ecflow-suite-to-complete@v2
+        with:
+          sbatch_options: |
+            #SBATCH --job-name=anemoi_test_monitor
+            #SBATCH --time=01:00:00
+            #SBATCH --qos=nf
+          site: hpc-batch
+          troika_user: ${{ secrets.troika_user }}
+          suite_name: anemoi_tests/${{ inputs.suite_name }}
+          ecflow_host: ${{ secrets.ecflow_host }}
+          ecflow_port: ${{ secrets.ecflow_port }}
+          delay: 60
+          timeout: 3600
+  print:
+    runs-on: hpc
+    name: Print
+    needs:
+      - deploy
+      - monitor
+    if: always() && needs.deploy.result == 'success'
+    steps:
+      - name: Print results
+        uses: ecmwf/reusable-workflows/ci-hpc-generic@v2
+        with:
+          template: |
+            set -eux
+            echo "Job is running on ${{ runner.os }}"
+
+            # Load modules of interest
+            module load wellies/1.2.0
+
+            mkdir -p /scratch/${{ secrets.troika_user }}/anemoi_tests/${{ inputs.suite_name }}
+            /home/mlx/bin/tracksuite-print /anemoi_tests/${{ inputs.suite_name }} --host ${{ secrets.ecflow_host }} -f html &> /scratch/${{ secrets.troika_user }}/anemoi_tests/${{ inputs.suite_name }}/summary.md
+          sbatch_options: |
+            #SBATCH --job-name=anemoi_test_print_results
+            #SBATCH --time=00:10:00
+            #SBATCH --qos=nf
+          site: hpc-batch
+          troika_user: ${{ secrets.troika_user }}
+  summary:
+    runs-on: hpc
+    name: Summary
+    needs: print
+    if: always() && needs.print.result == 'success'
+    steps:
+      - name: Print summary
+        run: |
+          scp ${{ secrets.troika_user }}@hpc-batch:/scratch/${{ secrets.troika_user }}/anemoi_tests/${{ inputs.suite_name }}/summary.md $GITHUB_STEP_SUMMARY
+  clean:
+    runs-on: hpc
+    name: Clean
+    needs:
+      - monitor
+      - summary
+    if: always() && needs.monitor.result == 'success'
+    steps:
+      - name: Clean the suite
+        uses: ecmwf/reusable-workflows/hpc/ecflow/remove-suite@v2
+        with:
+          sbatch_options: |
+            #SBATCH --job-name=anemoi_test_clean
+            #SBATCH --time=00:10:00
+            #SBATCH --qos=nf
+          site: hpc-batch
+          troika_user: ${{ secrets.troika_user }}
+          suite_name: anemoi_tests/${{ inputs.suite_name }}
+          ecflow_host: ${{ secrets.ecflow_host }}
+          ecflow_port: ${{ secrets.ecflow_port }}

.gitignore

@@ -3,3 +3,4 @@ _build/
 ?.*
 ~*
 *.egg-info/
+__pycache__/