From 0656f4ad3e11d984d5655cdea647f364bcefdba5 Mon Sep 17 00:00:00 2001 From: tombch Date: Wed, 4 Mar 2026 13:32:00 +0100 Subject: [PATCH 01/10] Added workflow to lint YAML files using yamlfmt --- .github/workflows/yamlfmt-lint.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/yamlfmt-lint.yml diff --git a/.github/workflows/yamlfmt-lint.yml b/.github/workflows/yamlfmt-lint.yml new file mode 100644 index 0000000000..a87cf039f8 --- /dev/null +++ b/.github/workflows/yamlfmt-lint.yml @@ -0,0 +1,20 @@ +name: yamlfmt-lint +on: + pull_request: + push: + branches: + - main + - yamlfmt-lint #ย For testing + workflow_dispatch: + +jobs: + lint: + runs-on: ubuntu-latest + container: + image: ghcr.io/google/yamlfmt:latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Run yamlfmt lint + run: yamlfmt -lint . From f05aa67a0996129d54f20391d84fc88504a07a9e Mon Sep 17 00:00:00 2001 From: tombch Date: Wed, 4 Mar 2026 14:16:25 +0100 Subject: [PATCH 02/10] Trim trailing whitespace setting preserves multi-line run/script tags in github actions, and retain single line breaks --- .yamlfmt.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.yamlfmt.yaml b/.yamlfmt.yaml index 8369992be0..a799fd26a5 100644 --- a/.yamlfmt.yaml +++ b/.yamlfmt.yaml @@ -2,6 +2,8 @@ formatter: type: basic drop_merge_tag: true + trim_trailing_whitespace: true + retain_line_breaks_single: true gitignore_excludes: true exclude: - kubernetes/loculus/templates From c60d8811a85d8dfde7d4ccc06e5ddeb64b62cae8 Mon Sep 17 00:00:00 2001 From: tombch Date: Wed, 4 Mar 2026 14:17:21 +0100 Subject: [PATCH 03/10] Formatted yaml files --- .github/dependabot.yml | 20 ++-- .github/workflows/add-preview-url.yml | 20 ++-- .github/workflows/backend-image.yml | 2 +- .github/workflows/claude-code-review.yml | 4 +- .github/workflows/cli-tests.yml | 8 +- .../workflows/datasets-mirror-priority-1.yml | 20 ++-- .../workflows/datasets-mirror-priority-2.yml | 60 +++++----- .github/workflows/datasets-mirror.yml | 2 +- .github/workflows/docs-image.yml | 2 +- .github/workflows/ena-submission-image.yaml | 2 +- .../workflows/ena-submission-unit-tests.yaml | 4 +- .../ena-submission-workflow-tests.yml | 23 ++-- .github/workflows/flyway-check.yml | 54 ++++----- .../workflows/format-backend-on-request.yml | 10 +- .../format-ena-deposition-on-dispatch.yml | 2 +- .../workflows/format-integration-tests.yml | 12 +- .../workflows/format-website-on-dispatch.yml | 2 +- .github/workflows/helm-schema-lint.yaml | 2 +- .github/workflows/ingest-image.yml | 2 +- .github/workflows/issue-triage.yml | 2 +- .github/workflows/keycloakify-image.yml | 2 +- .../workflows/preprocessing-dummy-image.yml | 2 +- .../preprocessing-nextclade-image.yml | 2 +- .github/workflows/schema-dump.yml | 108 ++++++++--------- .github/workflows/update-argocd-metadata.yml | 2 +- .github/workflows/website-image.yml | 2 +- .pre-commit-config.yaml | 70 +++++------ ena-submission/environment.yml | 2 +- ena-submission/test/test_config.yaml | 6 +- ingest/config/defaults.yaml | 6 +- ingest/environment.yml | 2 +- kubernetes/loculus/values.yaml | 110 +++++++++--------- kubernetes/loculus/values_e2e_and_dev.yaml | 2 +- kubernetes/loculus/values_preview_server.yaml | 2 +- preprocessing/dummy/lineage-alternative.yaml | 18 +-- preprocessing/dummy/lineage.yaml | 8 +- .../tests/embl_required_metadata.yaml | 2 +- .../tests/multi_reference_config.yaml | 16 +-- .../nextclade/tests/multi_segment_config.yaml | 8 +- .../tests/multi_segment_multi_reference.yaml | 24 ++-- .../nextclade/tests/no_alignment_config.yaml | 2 +- .../tests/single_segment_config.yaml | 4 +- 42 files changed, 320 insertions(+), 333 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fb7ce63576..a0fec3164f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,16 +9,16 @@ updates: schedule: interval: weekly ignore: - - dependency-name: "@tanstack/react-query" - # see https://github.com/loculus-project/loculus/issues/3305 - # see https://github.com/loculus-project/loculus/issues/3295 - versions: [ ">=5.0.0" ] - - dependency-name: "@mui/material" - - dependency-name: "change-case" - # see https://github.com/loculus-project/loculus/issues/4020 - - dependency-name: "flowbite-react" - - dependency-name: "@types/node" - versions: [">=23"] # Increase when we update node version in .nvmrc + - dependency-name: "@tanstack/react-query" + # see https://github.com/loculus-project/loculus/issues/3305 + # see https://github.com/loculus-project/loculus/issues/3295 + versions: [">=5.0.0"] + - dependency-name: "@mui/material" + - dependency-name: "change-case" + # see https://github.com/loculus-project/loculus/issues/4020 + - dependency-name: "flowbite-react" + - dependency-name: "@types/node" + versions: [">=23"] # Increase when we update node version in .nvmrc groups: minor: update-types: diff --git a/.github/workflows/add-preview-url.yml b/.github/workflows/add-preview-url.yml index 563cd72154..26276763ae 100644 --- a/.github/workflows/add-preview-url.yml +++ b/.github/workflows/add-preview-url.yml @@ -8,18 +8,18 @@ jobs: update-pr-description: runs-on: ubuntu-latest # This workflow uses pull_request_target to access repo secrets in PRs from forks - # SECURITY: This workflow only uses PR metadata (labels, title, etc.) + # SECURITY: This workflow only uses PR metadata (labels, title, etc.) # and does not check out any code from the fork, making it safe steps: - name: Update PR Description with Preview URL uses: actions/github-script@v8 with: github-token: ${{ secrets.LOCULUS_BOT_WRITE_PRS }} - script: | + script: |- const pr = context.payload.pull_request; const branchName = pr.head.ref; const hasPreviewLabel = pr.labels.some(label => label.name === 'preview'); - + // Format branch name exactly as in ArgoCD const shortBranchName = branchName.substring(0, 25) .replace(/_/g, "-") @@ -27,24 +27,24 @@ jobs: .replace(/\./g, "-") .replace(/-+$/, "") .toLowerCase(); - + const previewUrl = `https://${shortBranchName}.loculus.org`; const prDesc = pr.body || ''; - + // Create preview line - const previewLine = hasPreviewLabel + const previewLine = hasPreviewLabel ? `๐Ÿš€ Preview: ${previewUrl}` : `๐Ÿš€ Preview: Add \`preview\` label to enable`; - + // Remove existing preview line if present const previewRegex = /๐Ÿš€ Preview:.*/g; const descWithoutPreview = prDesc.replace(previewRegex, '').trim(); - + // Add new preview line - const updatedDesc = descWithoutPreview + const updatedDesc = descWithoutPreview ? `${descWithoutPreview}\n\n${previewLine}` : previewLine; - + // Update PR description await github.rest.pulls.update({ owner: context.repo.owner, diff --git a/.github/workflows/backend-image.yml b/.github/workflows/backend-image.yml index 252199bd64..424f29f3e3 100644 --- a/.github/workflows/backend-image.yml +++ b/.github/workflows/backend-image.yml @@ -101,7 +101,7 @@ jobs: platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} - name: Retag and push existing image if cache hit if: env.CACHE_HIT == 'true' - run: | + run: |- TAGS=(${{ steps.dockerMetadata.outputs.tags }}) for TAG in "${TAGS[@]}"; do docker buildx imagetools create --tag $TAG ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index 4732ed17c9..f19e32ada1 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -64,12 +64,12 @@ jobs: Provide detailed feedback using inline comments for specific issues. Use top-level comments for general observations or praise. - Generally if you have mentioned something in an inline comment you do not need + Generally if you have mentioned something in an inline comment you do not need to mention it again in the overall review. The overall review should be brief with most points made as inline comments. Keep language simple and to the point. Do not add inline comments that are simply praise - inline comments should be actionable. - claude_args: | + claude_args: |- --allowedTools "mcp__github_inline_comment__create_inline_comment,Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*)" diff --git a/.github/workflows/cli-tests.yml b/.github/workflows/cli-tests.yml index 3cc0bb5af4..bba3c05bae 100644 --- a/.github/workflows/cli-tests.yml +++ b/.github/workflows/cli-tests.yml @@ -27,9 +27,9 @@ jobs: - name: Install dependencies and package run: uv sync --all-extras working-directory: cli - # - name: Run tests - # run: uv run pytest - # working-directory: cli + # - name: Run tests + # run: uv run pytest + # working-directory: cli check-format: name: Check formatting runs-on: ubuntu-latest @@ -68,4 +68,4 @@ jobs: working-directory: cli - name: Check types with mypy run: uv run mypy . - working-directory: cli \ No newline at end of file + working-directory: cli diff --git a/.github/workflows/datasets-mirror-priority-1.yml b/.github/workflows/datasets-mirror-priority-1.yml index 4611ec00df..f5008b59aa 100644 --- a/.github/workflows/datasets-mirror-priority-1.yml +++ b/.github/workflows/datasets-mirror-priority-1.yml @@ -1,6 +1,6 @@ name: Mirror NCBI Datasets (Priority 1 - Every 2 Hours) on: - workflow_dispatch: # Allows manual triggering to re-run all priority-1 taxa + workflow_dispatch: # Allows manual triggering to re-run all priority-1 taxa schedule: - cron: '12 */2 * * *' # Runs every two hours at 12 minutes past the hour jobs: @@ -8,15 +8,15 @@ jobs: strategy: matrix: taxon_id: - - 10244 # Mpox - - 1868215 # Orthopneumovirus - - 3048148 # Metapneumovirus hominis - - 3048448 # West nile virus - - 3052345 # Morbillivirus hominis - - 3052460 # Orthoebolavirus sudanense - - 3052462 # Orthoebolavirus zairense - - 3052505 # Orthomarburgvirus marburgense - - 3052518 # CCHFV + - 10244 # Mpox + - 1868215 # Orthopneumovirus + - 3048148 # Metapneumovirus hominis + - 3048448 # West nile virus + - 3052345 # Morbillivirus hominis + - 3052460 # Orthoebolavirus sudanense + - 3052462 # Orthoebolavirus zairense + - 3052505 # Orthomarburgvirus marburgense + - 3052518 # CCHFV fail-fast: false uses: ./.github/workflows/datasets-mirror.yml secrets: inherit diff --git a/.github/workflows/datasets-mirror-priority-2.yml b/.github/workflows/datasets-mirror-priority-2.yml index 252afaf6fb..222051754f 100644 --- a/.github/workflows/datasets-mirror-priority-2.yml +++ b/.github/workflows/datasets-mirror-priority-2.yml @@ -1,6 +1,6 @@ name: Mirror NCBI Datasets (Priority 2 - Daily) on: - workflow_dispatch: # Allows manual triggering to re-run all priority-2 taxa + workflow_dispatch: # Allows manual triggering to re-run all priority-2 taxa schedule: - cron: '12 3 * * *' # Runs daily at 03:12 UTC jobs: @@ -8,35 +8,35 @@ jobs: strategy: matrix: taxon_id: - - 11053 # Dengue type 1 - - 11060 # Dengue type 2 - - 11069 # Dengue type 3 - - 11070 # Dengue type 4 - - 11137 # Coronavirus 229E - - 11234 # Measles morbillivirus - - 11250 # RSV - - 11520 # Influenza B - - 11676 # HIV1 - - 12059 # Enterovirus - - 12637 # Dengue (all) - - 31631 # Coronavirus OC43 - - 31704 # Coxsackievirus A16 - - 39054 # Enterovirus A71 - - 42769 # Coxsackievirus A10 - - 42789 # Enterovirus D68 - - 86107 # Coxsackievirus A6 - - 118655 # Oropouche - - 138948 # Enterovirus A - - 138949 # Enterovirus B - - 138950 # Enterovirus C - - 138951 # Enterovirus D - - 162145 # Human metapneumovirus - - 197911 # Influenza A - - 208893 # RSV A - - 208895 # RSV B - - 277944 # Coronavirus NL63 - - 290028 # Coronavirus HKU1 - - 3050294 # Chickenpox virus + - 11053 # Dengue type 1 + - 11060 # Dengue type 2 + - 11069 # Dengue type 3 + - 11070 # Dengue type 4 + - 11137 # Coronavirus 229E + - 11234 # Measles morbillivirus + - 11250 # RSV + - 11520 # Influenza B + - 11676 # HIV1 + - 12059 # Enterovirus + - 12637 # Dengue (all) + - 31631 # Coronavirus OC43 + - 31704 # Coxsackievirus A16 + - 39054 # Enterovirus A71 + - 42769 # Coxsackievirus A10 + - 42789 # Enterovirus D68 + - 86107 # Coxsackievirus A6 + - 118655 # Oropouche + - 138948 # Enterovirus A + - 138949 # Enterovirus B + - 138950 # Enterovirus C + - 138951 # Enterovirus D + - 162145 # Human metapneumovirus + - 197911 # Influenza A + - 208893 # RSV A + - 208895 # RSV B + - 277944 # Coronavirus NL63 + - 290028 # Coronavirus HKU1 + - 3050294 # Chickenpox virus fail-fast: false uses: ./.github/workflows/datasets-mirror.yml secrets: inherit diff --git a/.github/workflows/datasets-mirror.yml b/.github/workflows/datasets-mirror.yml index c37d6f2e05..7aca667b43 100644 --- a/.github/workflows/datasets-mirror.yml +++ b/.github/workflows/datasets-mirror.yml @@ -58,7 +58,7 @@ jobs: - name: Send Slack Notification env: SLACK_HOOK: ${{ secrets.SLACK_HOOK }} - run: | + run: |- curl -X POST -H 'Content-type: application/json' --data '{ "text": "๐Ÿšจ Failed to mirror NCBI datasets: ${{ github.repository }}\n๐Ÿ”— <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Details>" }' $SLACK_HOOK diff --git a/.github/workflows/docs-image.yml b/.github/workflows/docs-image.yml index d188316aef..cd29a78e7b 100644 --- a/.github/workflows/docs-image.yml +++ b/.github/workflows/docs-image.yml @@ -89,7 +89,7 @@ jobs: build-args: NODE_VERSION=${{ env.NODE_VERSION }} - name: Retag and push existing image if cache hit if: env.CACHE_HIT == 'true' - run: | + run: |- TAGS=(${{ steps.dockerMetadata.outputs.tags }}) for TAG in "${TAGS[@]}"; do docker buildx imagetools create --tag $TAG ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} diff --git a/.github/workflows/ena-submission-image.yaml b/.github/workflows/ena-submission-image.yaml index f3cba51e95..1696f1c606 100644 --- a/.github/workflows/ena-submission-image.yaml +++ b/.github/workflows/ena-submission-image.yaml @@ -100,7 +100,7 @@ jobs: done # Run smoke test to ensure the image is functional - name: Run smoke test (unit tests in docker) - run: | + run: |- docker run \ --rm ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} \ pytest scripts/test_ena_submission.py diff --git a/.github/workflows/ena-submission-unit-tests.yaml b/.github/workflows/ena-submission-unit-tests.yaml index 70cebce98f..3e8132ef82 100644 --- a/.github/workflows/ena-submission-unit-tests.yaml +++ b/.github/workflows/ena-submission-unit-tests.yaml @@ -58,9 +58,9 @@ jobs: cache-environment: true - name: Install package run: | - uv pip install --system . + uv pip install --system . - name: Run tests - run: | + run: |- # The integration tests are run in a separate workflow # They require postgres to be running pytest \ diff --git a/.github/workflows/ena-submission-workflow-tests.yml b/.github/workflows/ena-submission-workflow-tests.yml index f039ae5717..0e89b97b67 100644 --- a/.github/workflows/ena-submission-workflow-tests.yml +++ b/.github/workflows/ena-submission-workflow-tests.yml @@ -25,10 +25,7 @@ jobs: ports: - 5432:5432 options: >- - --health-cmd "pg_isready -U postgres" - --health-interval 10s - --health-timeout 5s - --health-retries 5 + --health-cmd "pg_isready -U postgres" --health-interval 10s --health-timeout 5s --health-retries 5 steps: - uses: actions/checkout@v6 - name: Extract Flyway version @@ -46,7 +43,7 @@ jobs: sudo mv flyway-$FLYWAY_VERSION /opt/flyway echo "/opt/flyway" >> $GITHUB_PATH - name: Run Flyway - run: | + run: | flyway -url=jdbc:postgresql://localhost:5432/loculus \ -schemas=ena_deposition_schema \ -user=postgres \ @@ -55,19 +52,19 @@ jobs: migrate - name: Set up micromamba uses: mamba-org/setup-micromamba@v2 - with: - environment-file: ena-submission/environment.yml - micromamba-version: 'latest' - cache-environment: true + with: + environment-file: ena-submission/environment.yml + micromamba-version: 'latest' + cache-environment: true - name: Install package run: | - uv pip install --system . + uv pip install --system . shell: micromamba-shell {0} working-directory: ena-submission - name: Run tests run: | - # The unit tests are run in ena-submission-unit-tests.yaml - pytest \ - --ignore=./scripts/test_ena_submission.py + # The unit tests are run in ena-submission-unit-tests.yaml + pytest \ + --ignore=./scripts/test_ena_submission.py shell: micromamba-shell {0} working-directory: ena-submission diff --git a/.github/workflows/flyway-check.yml b/.github/workflows/flyway-check.yml index 2804493412..6b50fc2c6c 100644 --- a/.github/workflows/flyway-check.yml +++ b/.github/workflows/flyway-check.yml @@ -2,38 +2,38 @@ name: Flyway Migration Check on: pull_request: - branches: [ main ] + branches: [main] paths: - 'backend/src/main/resources/db/migration/**' jobs: check-migrations: runs-on: ubuntu-latest - + steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Check for modified migrations + run: |- + git fetch origin main:main + modified_files=$(git diff --name-only main...HEAD -- src/main/resources/db/migration/) + new_files=$(git diff --name-only --diff-filter=A main...HEAD -- src/main/resources/db/migration/) + + if [ -n "$modified_files" ]; then + echo "The following existing migration files have been modified:" + echo "$modified_files" + echo "Error: Modifying existing migrations is not allowed." + exit 1 + fi + + if [ -n "$new_files" ]; then + echo "The following new migration files have been added:" + echo "$new_files" + echo "New migrations are allowed." + fi - - name: Check for modified migrations - run: | - git fetch origin main:main - modified_files=$(git diff --name-only main...HEAD -- src/main/resources/db/migration/) - new_files=$(git diff --name-only --diff-filter=A main...HEAD -- src/main/resources/db/migration/) - - if [ -n "$modified_files" ]; then - echo "The following existing migration files have been modified:" - echo "$modified_files" - echo "Error: Modifying existing migrations is not allowed." - exit 1 - fi - - if [ -n "$new_files" ]; then - echo "The following new migration files have been added:" - echo "$new_files" - echo "New migrations are allowed." - fi - - if [ -z "$modified_files" ] && [ -z "$new_files" ]; then - echo "No changes to migration files detected." - fi + if [ -z "$modified_files" ] && [ -z "$new_files" ]; then + echo "No changes to migration files detected." + fi diff --git a/.github/workflows/format-backend-on-request.yml b/.github/workflows/format-backend-on-request.yml index 22bab866da..1c1a518031 100644 --- a/.github/workflows/format-backend-on-request.yml +++ b/.github/workflows/format-backend-on-request.yml @@ -17,22 +17,22 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ github.head_ref }} - + - name: Set up JDK uses: actions/setup-java@v5 with: java-version: "21" distribution: "adopt" - + - name: Setup Gradle uses: gradle/actions/setup-gradle@v5 - + - name: Format Backend Code run: ./gradlew ktlintFormat working-directory: ./backend - + - name: Commit and Push Changes - run: | + run: |- git config --global user.name 'Loculus bot' git config --global user.email 'bot@loculus.org' git add -A diff --git a/.github/workflows/format-ena-deposition-on-dispatch.yml b/.github/workflows/format-ena-deposition-on-dispatch.yml index 69b922349e..d39d8a180c 100644 --- a/.github/workflows/format-ena-deposition-on-dispatch.yml +++ b/.github/workflows/format-ena-deposition-on-dispatch.yml @@ -34,7 +34,7 @@ jobs: shell: micromamba-shell {0} working-directory: ena-submission - name: Commit and Push Changes - run: | + run: |- git config --global user.name 'Loculus bot' git config --global user.email 'bot@loculus.org' git add -A diff --git a/.github/workflows/format-integration-tests.yml b/.github/workflows/format-integration-tests.yml index 86c61a6abf..05bc3ca338 100644 --- a/.github/workflows/format-integration-tests.yml +++ b/.github/workflows/format-integration-tests.yml @@ -11,23 +11,23 @@ on: jobs: format: runs-on: ubuntu-latest - + defaults: run: working-directory: ./integration-tests - + steps: - uses: actions/checkout@v6 - + - name: Setup Node.js uses: actions/setup-node@v6 with: node-version: '20' cache: 'npm' cache-dependency-path: integration-tests/package-lock.json - + - name: Install dependencies run: npm ci - + - name: Check formatting - run: npm run format:check \ No newline at end of file + run: npm run format:check diff --git a/.github/workflows/format-website-on-dispatch.yml b/.github/workflows/format-website-on-dispatch.yml index d23e7978ee..7e9d9d97ac 100644 --- a/.github/workflows/format-website-on-dispatch.yml +++ b/.github/workflows/format-website-on-dispatch.yml @@ -33,7 +33,7 @@ jobs: npm ci npm run format - name: Commit and Push Changes - run: | + run: |- git config --global user.name 'Loculus bot' git config --global user.email 'bot@loculus.org' git add -A diff --git a/.github/workflows/helm-schema-lint.yaml b/.github/workflows/helm-schema-lint.yaml index b3aeb47e2e..b8d6e86f1a 100644 --- a/.github/workflows/helm-schema-lint.yaml +++ b/.github/workflows/helm-schema-lint.yaml @@ -23,7 +23,7 @@ jobs: version: v3.18.3 - name: Run Helm lint on values files - run: | + run: |- helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml -f kubernetes/loculus/values_e2e_and_dev.yaml helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml -f kubernetes/loculus/values_preview_server.yaml diff --git a/.github/workflows/ingest-image.yml b/.github/workflows/ingest-image.yml index 055c25c5d1..6ad55d79c0 100644 --- a/.github/workflows/ingest-image.yml +++ b/.github/workflows/ingest-image.yml @@ -80,7 +80,7 @@ jobs: platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} - name: Retag and push existing image if cache hit if: env.CACHE_HIT == 'true' - run: | + run: |- TAGS=(${{ steps.dockerMetadata.outputs.tags }}) for TAG in "${TAGS[@]}"; do docker buildx imagetools create --tag $TAG ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml index c629940fe6..080cd9a07c 100644 --- a/.github/workflows/issue-triage.yml +++ b/.github/workflows/issue-triage.yml @@ -168,5 +168,5 @@ jobs: - Be conservative when suggesting related issues - Do not duplicate information already in the PR description - claude_args: | + claude_args: |- --allowedTools "Bash(gh pr:*),Bash(gh issue list:*),Bash(gh label list)" diff --git a/.github/workflows/keycloakify-image.yml b/.github/workflows/keycloakify-image.yml index 018de7ca67..b6db401d8f 100644 --- a/.github/workflows/keycloakify-image.yml +++ b/.github/workflows/keycloakify-image.yml @@ -87,7 +87,7 @@ jobs: build-args: NODE_VERSION=${{ steps.get-node-version.outputs.NODE_VERSION }} - name: Tag and push image if cache hit if: env.CACHE_HIT == 'true' - run: | + run: |- TAGS=(${{ steps.dockerMetadata.outputs.tags }}) for TAG in "${TAGS[@]}"; do docker buildx imagetools create --tag $TAG ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} diff --git a/.github/workflows/preprocessing-dummy-image.yml b/.github/workflows/preprocessing-dummy-image.yml index 781deabe1a..204ed790ab 100644 --- a/.github/workflows/preprocessing-dummy-image.yml +++ b/.github/workflows/preprocessing-dummy-image.yml @@ -79,7 +79,7 @@ jobs: platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} - name: Tag and push image if cache hit if: env.CACHE_HIT == 'true' - run: | + run: |- TAGS=(${{ steps.dockerMetadata.outputs.tags }}) for TAG in "${TAGS[@]}"; do docker buildx imagetools create --tag $TAG ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} diff --git a/.github/workflows/preprocessing-nextclade-image.yml b/.github/workflows/preprocessing-nextclade-image.yml index 98f9d7beba..a4fd8c5207 100644 --- a/.github/workflows/preprocessing-nextclade-image.yml +++ b/.github/workflows/preprocessing-nextclade-image.yml @@ -80,7 +80,7 @@ jobs: platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} - name: Retag and push existing image if cache hit if: env.CACHE_HIT == 'true' - run: | + run: |- TAGS=(${{ steps.dockerMetadata.outputs.tags }}) for TAG in "${TAGS[@]}"; do docker buildx imagetools create --tag $TAG ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} diff --git a/.github/workflows/schema-dump.yml b/.github/workflows/schema-dump.yml index a49e61fbbb..642607b861 100644 --- a/.github/workflows/schema-dump.yml +++ b/.github/workflows/schema-dump.yml @@ -11,7 +11,7 @@ on: jobs: generate-schema: runs-on: ubuntu-latest - + # Explicitly set permissions needed for committing permissions: contents: write @@ -25,69 +25,59 @@ jobs: POSTGRES_DB: postgres POSTGRES_USER: postgres options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 + --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 ports: - 5432:5432 steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - ref: ${{ github.head_ref }} - fetch-depth: 0 + - name: Checkout repository + uses: actions/checkout@v6 + with: + ref: ${{ github.head_ref }} + fetch-depth: 0 + + - name: Run Flyway migrations + uses: docker://flyway/flyway:10-alpine + env: + FLYWAY_URL: jdbc:postgresql://postgres:5432/postgres + FLYWAY_USER: postgres + FLYWAY_PASSWORD: postgres + with: + args: -locations=filesystem:./backend/src/main/resources/db/migration/ migrate - - name: Run Flyway migrations - uses: docker://flyway/flyway:10-alpine - env: - FLYWAY_URL: jdbc:postgresql://postgres:5432/postgres - FLYWAY_USER: postgres - FLYWAY_PASSWORD: postgres - with: - args: -locations=filesystem:./backend/src/main/resources/db/migration/ migrate + - name: Generate new schema + uses: docker://postgres:16 + with: + args: > + bash -c "PGPASSWORD=postgres pg_dump -h postgres -U postgres -d postgres --schema-only --no-security-labels --restrict-key='dummy' > /github/workspace/backend/docs/db/schema.sql" + - name: Cat schema + run: cat backend/docs/db/schema.sql - - name: Generate new schema - uses: docker://postgres:16 - with: - args: > - bash -c "PGPASSWORD=postgres pg_dump - -h postgres - -U postgres - -d postgres - --schema-only - --no-security-labels - --restrict-key='dummy' - > /github/workspace/backend/docs/db/schema.sql" - - name: Cat schema - run: cat backend/docs/db/schema.sql - - - name: Stage schema file - run: git add backend/docs/db/schema.sql + - name: Stage schema file + run: git add backend/docs/db/schema.sql - - name: Check for schema changes - id: check-changes - run: | - if ! git diff --cached --quiet backend/docs/db/schema.sql; then - echo "Schema changes detected" - echo "changed=true" >> $GITHUB_OUTPUT - git diff --cached backend/docs/db/schema.sql - else - echo "No schema changes detected" - echo "changed=false" >> $GITHUB_OUTPUT - fi + - name: Check for schema changes + id: check-changes + run: | + if ! git diff --cached --quiet backend/docs/db/schema.sql; then + echo "Schema changes detected" + echo "changed=true" >> $GITHUB_OUTPUT + git diff --cached backend/docs/db/schema.sql + else + echo "No schema changes detected" + echo "changed=false" >> $GITHUB_OUTPUT + fi - - name: Handle schema changes - if: steps.check-changes.outputs.changed == 'true' - run: | - if [[ "${{ contains(github.event.pull_request.labels.*.name, 'update_db_schema') }}" == "true" ]]; then - git config --global user.name 'GitHub Action' - git config --global user.email 'action@github.com' - git commit -m "Update schema documentation based on migration changes" - git push - else - echo "::error::Schema changes detected but 'update-schema' label is not present on the PR" - echo "Please add the 'update_db_schema' label if these changes are intentional" - exit 1 - fi + - name: Handle schema changes + if: steps.check-changes.outputs.changed == 'true' + run: |- + if [[ "${{ contains(github.event.pull_request.labels.*.name, 'update_db_schema') }}" == "true" ]]; then + git config --global user.name 'GitHub Action' + git config --global user.email 'action@github.com' + git commit -m "Update schema documentation based on migration changes" + git push + else + echo "::error::Schema changes detected but 'update-schema' label is not present on the PR" + echo "Please add the 'update_db_schema' label if these changes are intentional" + exit 1 + fi diff --git a/.github/workflows/update-argocd-metadata.yml b/.github/workflows/update-argocd-metadata.yml index 5b44ea9914..2fdc7c797e 100644 --- a/.github/workflows/update-argocd-metadata.yml +++ b/.github/workflows/update-argocd-metadata.yml @@ -106,7 +106,7 @@ jobs: "head_short_sha_7": "${{ steps.get_sha.outputs.sha }}" }' > config.json - name: Commit and Push Changes - run: | + run: |- git config --global user.name 'Loculus bot' git config --global user.email 'bot@loculus.org' git add config.json diff --git a/.github/workflows/website-image.yml b/.github/workflows/website-image.yml index 8d45537275..0800fcdae0 100644 --- a/.github/workflows/website-image.yml +++ b/.github/workflows/website-image.yml @@ -156,7 +156,7 @@ jobs: docker buildx imagetools inspect ${{ env.DOCKER_IMAGE_NAME }}:${{ needs.check-cache.outputs.dir-hash }} - name: Retag and push existing image if: needs.check-cache.outputs.cache-hit == 'true' - run: | + run: |- METADATA='${{ steps.dockerMetadata.outputs.json }}' TAGS=$(jq -r '.tags[]' <<< "$METADATA") for TAG in $TAGS; do diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df4e073db1..da1b4ed811 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,36 +1,36 @@ repos: -- repo: https://github.com/astral-sh/ruff-pre-commit - # Ruff version. - rev: v0.14.2 - hooks: - # Run the linter and fix issues. - - id: ruff-check - args: [ --fix ] - files: ^ena-submission/ - # Run lint but only fix auto-fixable issues. - - id: ruff-check - args: [ --fix-only ] - files: ^preprocessing/nextclade/ - # Run the formatter. - - id: ruff-format - files: ^ena-submission/|^preprocessing/nextclade/ -- repo: local - hooks: - - id: prettier-values-schema - name: prettier (values.schema.json) - entry: npx prettier@3.6.2 --write - language: system - files: ^kubernetes/loculus/values\.schema\.json$ - - id: helm-lint - name: helm lint - entry: bash - args: - - -c - - | - set -euo pipefail - helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml - helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml -f kubernetes/loculus/values_e2e_and_dev.yaml - helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml -f kubernetes/loculus/values_preview_server.yaml - language: system - pass_filenames: false - files: ^kubernetes/loculus/ + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.14.2 + hooks: + # Run the linter and fix issues. + - id: ruff-check + args: [--fix] + files: ^ena-submission/ + # Run lint but only fix auto-fixable issues. + - id: ruff-check + args: [--fix-only] + files: ^preprocessing/nextclade/ + # Run the formatter. + - id: ruff-format + files: ^ena-submission/|^preprocessing/nextclade/ + - repo: local + hooks: + - id: prettier-values-schema + name: prettier (values.schema.json) + entry: npx prettier@3.6.2 --write + language: system + files: ^kubernetes/loculus/values\.schema\.json$ + - id: helm-lint + name: helm lint + entry: bash + args: + - -c + - | + set -euo pipefail + helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml + helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml -f kubernetes/loculus/values_e2e_and_dev.yaml + helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml -f kubernetes/loculus/values_preview_server.yaml + language: system + pass_filenames: false + files: ^kubernetes/loculus/ diff --git a/ena-submission/environment.yml b/ena-submission/environment.yml index 5d3c4910c4..5556e4efbc 100644 --- a/ena-submission/environment.yml +++ b/ena-submission/environment.yml @@ -5,7 +5,7 @@ channels: - nodefaults dependencies: # Core Python dependencies - - python=3.12.9 # Pinned to 3.12.x series, check for patch updates within 3.12 + - python=3.12.9 # Pinned to 3.12.x series, check for patch updates within 3.12 - pip=25.2 - uv=0.9.5 # Extra dependencies diff --git a/ena-submission/test/test_config.yaml b/ena-submission/test/test_config.yaml index 289f63ff2a..a3759cc452 100644 --- a/ena-submission/test/test_config.yaml +++ b/ena-submission/test/test_config.yaml @@ -19,7 +19,7 @@ enaOrganisms: scientific_name: Orthonairovirus haemorrhagiae taxon_id: 3052518 organismName: "Crimean-Congo Hemorrhagic Fever Virus" - segments: + segments: - L - M - S @@ -92,7 +92,7 @@ enaOrganisms: scientific_name: West Nile virus taxon_id: 11082 organismName: "West Nile Virus" - segments: + segments: - main externalMetadata: - externalMetadataUpdater: ena @@ -133,4 +133,4 @@ enaOrganisms: type: int - externalMetadataUpdater: ena name: sraRunAccession - type: string \ No newline at end of file + type: string diff --git a/ingest/config/defaults.yaml b/ingest/config/defaults.yaml index 308dd12396..7296e01b30 100644 --- a/ingest/config/defaults.yaml +++ b/ingest/config/defaults.yaml @@ -62,16 +62,16 @@ ncbi_mappings: - ncbiHostCommonName - ncbiPurposeOfSampling - ncbiHostSex -group_name: insdc_ingest_group # Used only to set the group name, never read +group_name: insdc_ingest_group # Used only to set the group name, never read username: insdc_ingest_user password: insdc_ingest_user keycloak_client_id: backend-client subsample_fraction: 1.0 -approve_timeout_min: "25" # Cronjobs run every 30min, make approve stop before it is forced to stop by argocd +approve_timeout_min: "25" # Cronjobs run every 30min, make approve stop before it is forced to stop by argocd db_username: postgres db_password: unsecure db_url: "jdbc:postgresql://127.0.0.1:5432/loculus" -batch_chunk_size: 10000 # Batch size for submitting sequences to Loculus backend +batch_chunk_size: 10000 # Batch size for submitting sequences to Loculus backend nextclade_dataset_server: https://data.clades.nextstrain.org/v3 time_between_approve_requests_seconds: 60 backend_request_timeout_seconds: 600 diff --git a/ingest/environment.yml b/ingest/environment.yml index 2ac91266e7..516f2322f5 100644 --- a/ingest/environment.yml +++ b/ingest/environment.yml @@ -5,7 +5,7 @@ channels: - nodefaults dependencies: # Core Python dependencies - - python=3.12.9 # Pinned to 3.12.x series, check for patch updates within 3.12 + - python=3.12.9 # Pinned to 3.12.x series, check for patch updates within 3.12 # Extra dependencies - biopython=1.85 - click=8.3.0 diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index 9b03b14a50..ac326b65d3 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -1391,8 +1391,8 @@ defaultOrganismConfig: &defaultOrganismConfig segments: - name: main references: - - name: singleReference - genes: [] + - name: singleReference + genes: [] nextclade_dataset_server: https://data.clades.nextstrain.org/v3 ingest: &ingest image: ghcr.io/loculus-project/ingest @@ -1415,9 +1415,9 @@ defaultOrganisms: segments: - name: main references: - - name: singleReference - nextclade_dataset_name: nextstrain/ebola/sudan - genes: [NP, VP35, VP40, GP, sGP, ssGP, VP30, VP24, L] + - name: singleReference + nextclade_dataset_name: nextstrain/ebola/sudan + genes: [NP, VP35, VP40, GP, sGP, ssGP, VP30, VP24, L] nextclade_dataset_server: https://raw.githubusercontent.com/nextstrain/nextclade_data/ebola/data_output taxon_id: 186540 scientific_name: "Sudan ebolavirus" @@ -1512,9 +1512,9 @@ defaultOrganisms: segments: - name: main references: - - name: singleReference - nextclade_dataset_name: nextstrain/wnv/all-lineages - genes: [capsid, prM, env, NS1, NS2A, NS2B, NS3, NS4A, 2K, NS4B, NS5] + - name: singleReference + nextclade_dataset_name: nextstrain/wnv/all-lineages + genes: [capsid, prM, env, NS1, NS2A, NS2B, NS3, NS4A, 2K, NS4B, NS5] nextclade_dataset_server: https://raw.githubusercontent.com/nextstrain/nextclade_data/wnv/data_output taxon_id: 11082 scientific_name: "West Nile virus" @@ -1611,7 +1611,7 @@ defaultOrganisms: required: true type: string lineageSystem: pangoLineage - options: + options: - name: A - name: A.1 - name: A.1.1 @@ -1723,7 +1723,7 @@ defaultOrganisms: required: true type: string lineageSystem: alternativeLineage - options: + options: - name: A - name: A.1 - name: A.1.1 @@ -1779,8 +1779,8 @@ defaultOrganisms: segments: - name: main references: - - name: singleReference - genes: [] + - name: singleReference + genes: [] referenceGenomes: - name: main references: @@ -1853,19 +1853,19 @@ defaultOrganisms: segments: - name: L references: - - nextclade_dataset_name: community/pathoplexus/cchfv/L - name: singleReference - genes: [RdRp] + - nextclade_dataset_name: community/pathoplexus/cchfv/L + name: singleReference + genes: [RdRp] - name: M references: - - nextclade_dataset_name: community/pathoplexus/cchfv/M - name: singleReference - genes: [GPC] + - nextclade_dataset_name: community/pathoplexus/cchfv/M + name: singleReference + genes: [GPC] - name: S references: - - nextclade_dataset_name: community/pathoplexus/cchfv/S - name: singleReference - genes: [NP] + - nextclade_dataset_name: community/pathoplexus/cchfv/S + name: singleReference + genes: [NP] ingest: <<: *ingest configFile: @@ -1990,32 +1990,32 @@ defaultOrganisms: segments: - name: L references: - - nextclade_dataset_name: cchfv/L - name: singleReference - genes: [RdRp] + - nextclade_dataset_name: cchfv/L + name: singleReference + genes: [RdRp] - name: M references: - - nextclade_dataset_name: cchfv/M-MH396653 - name: MH396653 - genes: [GPC] - - nextclade_dataset_name: cchfv/M-OR047158 - name: OR047158 - genes: [GPC] + - nextclade_dataset_name: cchfv/M-MH396653 + name: MH396653 + genes: [GPC] + - nextclade_dataset_name: cchfv/M-OR047158 + name: OR047158 + genes: [GPC] - name: S references: - - nextclade_dataset_name: cchfv/S-1and6 - name: 1and6 - genes: [NP] - - nextclade_dataset_name: cchfv/S-2to5 - name: 2to5 - genes: [NP] + - nextclade_dataset_name: cchfv/S-1and6 + name: 1and6 + genes: [NP] + - nextclade_dataset_name: cchfv/S-2to5 + name: 2to5 + genes: [NP] ingest: <<: *ingest configFile: taxon_id: 3052518 segment_identification: method: "align" - nextclade_dataset_name: community/pathoplexus/cchfv #TODO: allow multiple ref in ingest grouping algo + nextclade_dataset_name: community/pathoplexus/cchfv #TODO: allow multiple ref in ingest grouping algo enaDeposition: singleReference: configFile: @@ -2164,23 +2164,23 @@ defaultOrganisms: segments: - name: main references: - - name: CV-A16 - nextclade_dataset_name: enpen/enterovirus/cv-a16 - accepted_dataset_matches: ["community/hodcroftlab/enterovirus/cva16", "community/hodcroftlab/enterovirus/enterovirus/linked/CV-A16"] - genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] - - name: CV-A10 - nextclade_dataset_name: enpen/enterovirus/cv-a10 - accepted_dataset_matches: ["community/hodcroftlab/enterovirus/enterovirus/linked/CV-A10"] - genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] - - name: EV-A71 - nextclade_dataset_name: enpen/enterovirus/ev-a71 - accepted_dataset_matches: ["community/hodcroftlab/enterovirus/enterovirus/linked/EV-A71"] - genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] - - name: EV-D68 - accepted_dataset_matches: ["community/hodcroftlab/enterovirus/enterovirus/linked/EV-D68"] - genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] - nextclade_dataset_name: enpen/enterovirus/ev-d68 - nextclade_dataset_server: https://data.clades.nextstrain.org/v3 + - name: CV-A16 + nextclade_dataset_name: enpen/enterovirus/cv-a16 + accepted_dataset_matches: ["community/hodcroftlab/enterovirus/cva16", "community/hodcroftlab/enterovirus/enterovirus/linked/CV-A16"] + genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] + - name: CV-A10 + nextclade_dataset_name: enpen/enterovirus/cv-a10 + accepted_dataset_matches: ["community/hodcroftlab/enterovirus/enterovirus/linked/CV-A10"] + genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] + - name: EV-A71 + nextclade_dataset_name: enpen/enterovirus/ev-a71 + accepted_dataset_matches: ["community/hodcroftlab/enterovirus/enterovirus/linked/EV-A71"] + genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] + - name: EV-D68 + accepted_dataset_matches: ["community/hodcroftlab/enterovirus/enterovirus/linked/EV-D68"] + genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] + nextclade_dataset_name: enpen/enterovirus/ev-d68 + nextclade_dataset_server: https://data.clades.nextstrain.org/v3 nextclade_dataset_server: https://raw.githubusercontent.com/nextstrain/nextclade_data/evs-datasets/data_output ingest: <<: *ingest @@ -2418,7 +2418,7 @@ enforceHTTPS: true registrationTermsMessage: > You must agree to the terms of use. -enaDeposition: +enaDeposition: submitToEnaProduction: false enaDbName: Loculus enaUniqueSuffix: Loculus diff --git a/kubernetes/loculus/values_e2e_and_dev.yaml b/kubernetes/loculus/values_e2e_and_dev.yaml index f7fce7b84d..032af7efee 100644 --- a/kubernetes/loculus/values_e2e_and_dev.yaml +++ b/kubernetes/loculus/values_e2e_and_dev.yaml @@ -14,4 +14,4 @@ auth: verifyEmail: false host: localhost:3000 siloImport: - pollIntervalSeconds: 5 \ No newline at end of file + pollIntervalSeconds: 5 diff --git a/kubernetes/loculus/values_preview_server.yaml b/kubernetes/loculus/values_preview_server.yaml index a7acaa3523..c6a1ea97d2 100644 --- a/kubernetes/loculus/values_preview_server.yaml +++ b/kubernetes/loculus/values_preview_server.yaml @@ -140,4 +140,4 @@ defaultResources: cpu: "20m" limits: memory: "1Gi" - cpu: "20m" \ No newline at end of file + cpu: "20m" diff --git a/preprocessing/dummy/lineage-alternative.yaml b/preprocessing/dummy/lineage-alternative.yaml index b2b34e1fa8..8f7008394d 100644 --- a/preprocessing/dummy/lineage-alternative.yaml +++ b/preprocessing/dummy/lineage-alternative.yaml @@ -4,30 +4,30 @@ A: A.1: aliases: [] parents: - - A + - A A.2: aliases: [] parents: - - A + - A A.1.1: aliases: [] parents: - - A.1 - - A.2 + - A.1 + - A.2 B: aliases: [] parents: [] B.1: aliases: [] parents: - - B + - B B.1.1: aliases: - - C + - C parents: - - B.1 + - B.1 C.1: aliases: - - B.1.1.1 + - B.1.1.1 parents: - - C + - C diff --git a/preprocessing/dummy/lineage.yaml b/preprocessing/dummy/lineage.yaml index 0c533e1a6c..52236c2360 100644 --- a/preprocessing/dummy/lineage.yaml +++ b/preprocessing/dummy/lineage.yaml @@ -4,13 +4,13 @@ A: A.1: aliases: [] parents: - - A + - A A.1.1: aliases: - - B + - B parents: - - A.1 + - A.1 A.2: aliases: [] parents: - - A + - A diff --git a/preprocessing/nextclade/tests/embl_required_metadata.yaml b/preprocessing/nextclade/tests/embl_required_metadata.yaml index 919e8f94e9..18641f04a9 100644 --- a/preprocessing/nextclade/tests/embl_required_metadata.yaml +++ b/preprocessing/nextclade/tests/embl_required_metadata.yaml @@ -20,4 +20,4 @@ processing_spec: type: date function: parse_and_assert_past_date inputs: - date: sampleCollectionDate \ No newline at end of file + date: sampleCollectionDate diff --git a/preprocessing/nextclade/tests/multi_reference_config.yaml b/preprocessing/nextclade/tests/multi_reference_config.yaml index 5784f1e403..f2f2aae0b9 100644 --- a/preprocessing/nextclade/tests/multi_reference_config.yaml +++ b/preprocessing/nextclade/tests/multi_reference_config.yaml @@ -7,14 +7,14 @@ nextclade_dataset_server: TEST segments: - name: main references: - - name: ebola-sudan - nextclade_dataset_name: ebola-dataset/ebola-sudan - accepted_dataset_matches: [ebola-sudan] - genes: [NPEbolaSudan, VP35EbolaSudan] # Names of genes in nextclade dataset gff3 file - - name: ebola-zaire - nextclade_dataset_name: ebola-dataset/ebola-zaire - accepted_dataset_matches: [ebola-zaire] - genes: [VP24EbolaZaire, LEbolaZaire] + - name: ebola-sudan + nextclade_dataset_name: ebola-dataset/ebola-sudan + accepted_dataset_matches: [ebola-sudan] + genes: [NPEbolaSudan, VP35EbolaSudan] # Names of genes in nextclade dataset gff3 file + - name: ebola-zaire + nextclade_dataset_name: ebola-dataset/ebola-zaire + accepted_dataset_matches: [ebola-zaire] + genes: [VP24EbolaZaire, LEbolaZaire] organism: multi-ebola-test processing_spec: subtype: diff --git a/preprocessing/nextclade/tests/multi_segment_config.yaml b/preprocessing/nextclade/tests/multi_segment_config.yaml index 2a9fa7a5b9..cfded28c62 100644 --- a/preprocessing/nextclade/tests/multi_segment_config.yaml +++ b/preprocessing/nextclade/tests/multi_segment_config.yaml @@ -6,12 +6,12 @@ segment_classification_method: "minimizer" segments: - name: ebola-sudan references: - - nextclade_dataset_name: ebola-dataset/ebola-sudan - genes: [NPEbolaSudan, VP35EbolaSudan] + - nextclade_dataset_name: ebola-dataset/ebola-sudan + genes: [NPEbolaSudan, VP35EbolaSudan] - name: ebola-zaire references: - - nextclade_dataset_name: ebola-dataset/ebola-zaire - genes: [VP24EbolaZaire, LEbolaZaire] + - nextclade_dataset_name: ebola-dataset/ebola-zaire + genes: [VP24EbolaZaire, LEbolaZaire] organism: multi-ebola-test processing_spec: totalInsertedNucs_ebola-zaire: diff --git a/preprocessing/nextclade/tests/multi_segment_multi_reference.yaml b/preprocessing/nextclade/tests/multi_segment_multi_reference.yaml index fea2a6e783..b5f69f9702 100644 --- a/preprocessing/nextclade/tests/multi_segment_multi_reference.yaml +++ b/preprocessing/nextclade/tests/multi_segment_multi_reference.yaml @@ -8,20 +8,20 @@ diamond_dmnd_url: FAKE_URL segments: - name: L references: - - name: singleReference - nextclade_dataset_name: CCHF/L - accepted_dataset_matches: [L] - genes: [RdRp] + - name: singleReference + nextclade_dataset_name: CCHF/L + accepted_dataset_matches: [L] + genes: [RdRp] - name: S references: - - name: 1and6 - nextclade_dataset_name: CCHF/S-1and6 - accepted_dataset_matches: [S-1and6] - genes: [NP] - - name: 2to5 - nextclade_dataset_name: CCHF/S-2to5 - accepted_dataset_matches: [S-2to5] - genes: [NP] + - name: 1and6 + nextclade_dataset_name: CCHF/S-1and6 + accepted_dataset_matches: [S-1and6] + genes: [NP] + - name: 2to5 + nextclade_dataset_name: CCHF/S-2to5 + accepted_dataset_matches: [S-2to5] + genes: [NP] organism: multi-reference-cchf-test processing_spec: subtype_S: diff --git a/preprocessing/nextclade/tests/no_alignment_config.yaml b/preprocessing/nextclade/tests/no_alignment_config.yaml index 569a649a5f..da2eb5c58c 100644 --- a/preprocessing/nextclade/tests/no_alignment_config.yaml +++ b/preprocessing/nextclade/tests/no_alignment_config.yaml @@ -87,7 +87,7 @@ processing_spec: extracted_regex_field: function: extract_regex args: - pattern: "^EPI_ISL_(?P[0-9]+)?$" # Note the optional group to allow for testing of missing values + pattern: "^EPI_ISL_(?P[0-9]+)?$" # Note the optional group to allow for testing of missing values capture_group: id inputs: regex_field: regex_field diff --git a/preprocessing/nextclade/tests/single_segment_config.yaml b/preprocessing/nextclade/tests/single_segment_config.yaml index 408f0beb41..3c8f31bcc4 100644 --- a/preprocessing/nextclade/tests/single_segment_config.yaml +++ b/preprocessing/nextclade/tests/single_segment_config.yaml @@ -8,8 +8,8 @@ db_name: "Loculus" segments: - name: main references: - - nextclade_dataset_name: ebola-sudan - genes: [NPEbolaSudan, VP35EbolaSudan] + - nextclade_dataset_name: ebola-sudan + genes: [NPEbolaSudan, VP35EbolaSudan] organism: ebola-sudan-test processing_spec: completeness: From ceea7e9c37219f2aea11ed3752c5f4bd60d711a3 Mon Sep 17 00:00:00 2001 From: tombch Date: Wed, 4 Mar 2026 14:21:24 +0100 Subject: [PATCH 04/10] Only runs on main branch and prs --- .github/workflows/yamlfmt-lint.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/yamlfmt-lint.yml b/.github/workflows/yamlfmt-lint.yml index a87cf039f8..23aed02536 100644 --- a/.github/workflows/yamlfmt-lint.yml +++ b/.github/workflows/yamlfmt-lint.yml @@ -4,7 +4,6 @@ on: push: branches: - main - - yamlfmt-lint #ย For testing workflow_dispatch: jobs: From 1e7901ab68e4a1961bd9aa8737c2326fcc541ac4 Mon Sep 17 00:00:00 2001 From: tombch Date: Wed, 4 Mar 2026 16:53:17 +0100 Subject: [PATCH 05/10] Revert "Only runs on main branch and prs" This reverts commit ceea7e9c37219f2aea11ed3752c5f4bd60d711a3. --- .github/workflows/yamlfmt-lint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/yamlfmt-lint.yml b/.github/workflows/yamlfmt-lint.yml index 23aed02536..a87cf039f8 100644 --- a/.github/workflows/yamlfmt-lint.yml +++ b/.github/workflows/yamlfmt-lint.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - yamlfmt-lint #ย For testing workflow_dispatch: jobs: From 1b3dac829e98d46090c79bdb98872d433eedf64d Mon Sep 17 00:00:00 2001 From: tombch Date: Wed, 4 Mar 2026 16:53:24 +0100 Subject: [PATCH 06/10] Revert "Formatted yaml files" This reverts commit c60d8811a85d8dfde7d4ccc06e5ddeb64b62cae8. --- .github/dependabot.yml | 20 ++-- .github/workflows/add-preview-url.yml | 20 ++-- .github/workflows/backend-image.yml | 2 +- .github/workflows/claude-code-review.yml | 4 +- .github/workflows/cli-tests.yml | 8 +- .../workflows/datasets-mirror-priority-1.yml | 20 ++-- .../workflows/datasets-mirror-priority-2.yml | 60 +++++----- .github/workflows/datasets-mirror.yml | 2 +- .github/workflows/docs-image.yml | 2 +- .github/workflows/ena-submission-image.yaml | 2 +- .../workflows/ena-submission-unit-tests.yaml | 4 +- .../ena-submission-workflow-tests.yml | 23 ++-- .github/workflows/flyway-check.yml | 54 ++++----- .../workflows/format-backend-on-request.yml | 10 +- .../format-ena-deposition-on-dispatch.yml | 2 +- .../workflows/format-integration-tests.yml | 12 +- .../workflows/format-website-on-dispatch.yml | 2 +- .github/workflows/helm-schema-lint.yaml | 2 +- .github/workflows/ingest-image.yml | 2 +- .github/workflows/issue-triage.yml | 2 +- .github/workflows/keycloakify-image.yml | 2 +- .../workflows/preprocessing-dummy-image.yml | 2 +- .../preprocessing-nextclade-image.yml | 2 +- .github/workflows/schema-dump.yml | 108 +++++++++-------- .github/workflows/update-argocd-metadata.yml | 2 +- .github/workflows/website-image.yml | 2 +- .pre-commit-config.yaml | 70 +++++------ ena-submission/environment.yml | 2 +- ena-submission/test/test_config.yaml | 6 +- ingest/config/defaults.yaml | 6 +- ingest/environment.yml | 2 +- kubernetes/loculus/values.yaml | 110 +++++++++--------- kubernetes/loculus/values_e2e_and_dev.yaml | 2 +- kubernetes/loculus/values_preview_server.yaml | 2 +- preprocessing/dummy/lineage-alternative.yaml | 18 +-- preprocessing/dummy/lineage.yaml | 8 +- .../tests/embl_required_metadata.yaml | 2 +- .../tests/multi_reference_config.yaml | 16 +-- .../nextclade/tests/multi_segment_config.yaml | 8 +- .../tests/multi_segment_multi_reference.yaml | 24 ++-- .../nextclade/tests/no_alignment_config.yaml | 2 +- .../tests/single_segment_config.yaml | 4 +- 42 files changed, 333 insertions(+), 320 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a0fec3164f..fb7ce63576 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,16 +9,16 @@ updates: schedule: interval: weekly ignore: - - dependency-name: "@tanstack/react-query" - # see https://github.com/loculus-project/loculus/issues/3305 - # see https://github.com/loculus-project/loculus/issues/3295 - versions: [">=5.0.0"] - - dependency-name: "@mui/material" - - dependency-name: "change-case" - # see https://github.com/loculus-project/loculus/issues/4020 - - dependency-name: "flowbite-react" - - dependency-name: "@types/node" - versions: [">=23"] # Increase when we update node version in .nvmrc + - dependency-name: "@tanstack/react-query" + # see https://github.com/loculus-project/loculus/issues/3305 + # see https://github.com/loculus-project/loculus/issues/3295 + versions: [ ">=5.0.0" ] + - dependency-name: "@mui/material" + - dependency-name: "change-case" + # see https://github.com/loculus-project/loculus/issues/4020 + - dependency-name: "flowbite-react" + - dependency-name: "@types/node" + versions: [">=23"] # Increase when we update node version in .nvmrc groups: minor: update-types: diff --git a/.github/workflows/add-preview-url.yml b/.github/workflows/add-preview-url.yml index 26276763ae..563cd72154 100644 --- a/.github/workflows/add-preview-url.yml +++ b/.github/workflows/add-preview-url.yml @@ -8,18 +8,18 @@ jobs: update-pr-description: runs-on: ubuntu-latest # This workflow uses pull_request_target to access repo secrets in PRs from forks - # SECURITY: This workflow only uses PR metadata (labels, title, etc.) + # SECURITY: This workflow only uses PR metadata (labels, title, etc.) # and does not check out any code from the fork, making it safe steps: - name: Update PR Description with Preview URL uses: actions/github-script@v8 with: github-token: ${{ secrets.LOCULUS_BOT_WRITE_PRS }} - script: |- + script: | const pr = context.payload.pull_request; const branchName = pr.head.ref; const hasPreviewLabel = pr.labels.some(label => label.name === 'preview'); - + // Format branch name exactly as in ArgoCD const shortBranchName = branchName.substring(0, 25) .replace(/_/g, "-") @@ -27,24 +27,24 @@ jobs: .replace(/\./g, "-") .replace(/-+$/, "") .toLowerCase(); - + const previewUrl = `https://${shortBranchName}.loculus.org`; const prDesc = pr.body || ''; - + // Create preview line - const previewLine = hasPreviewLabel + const previewLine = hasPreviewLabel ? `๐Ÿš€ Preview: ${previewUrl}` : `๐Ÿš€ Preview: Add \`preview\` label to enable`; - + // Remove existing preview line if present const previewRegex = /๐Ÿš€ Preview:.*/g; const descWithoutPreview = prDesc.replace(previewRegex, '').trim(); - + // Add new preview line - const updatedDesc = descWithoutPreview + const updatedDesc = descWithoutPreview ? `${descWithoutPreview}\n\n${previewLine}` : previewLine; - + // Update PR description await github.rest.pulls.update({ owner: context.repo.owner, diff --git a/.github/workflows/backend-image.yml b/.github/workflows/backend-image.yml index 424f29f3e3..252199bd64 100644 --- a/.github/workflows/backend-image.yml +++ b/.github/workflows/backend-image.yml @@ -101,7 +101,7 @@ jobs: platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} - name: Retag and push existing image if cache hit if: env.CACHE_HIT == 'true' - run: |- + run: | TAGS=(${{ steps.dockerMetadata.outputs.tags }}) for TAG in "${TAGS[@]}"; do docker buildx imagetools create --tag $TAG ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index f19e32ada1..4732ed17c9 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -64,12 +64,12 @@ jobs: Provide detailed feedback using inline comments for specific issues. Use top-level comments for general observations or praise. - Generally if you have mentioned something in an inline comment you do not need + Generally if you have mentioned something in an inline comment you do not need to mention it again in the overall review. The overall review should be brief with most points made as inline comments. Keep language simple and to the point. Do not add inline comments that are simply praise - inline comments should be actionable. - claude_args: |- + claude_args: | --allowedTools "mcp__github_inline_comment__create_inline_comment,Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*)" diff --git a/.github/workflows/cli-tests.yml b/.github/workflows/cli-tests.yml index bba3c05bae..3cc0bb5af4 100644 --- a/.github/workflows/cli-tests.yml +++ b/.github/workflows/cli-tests.yml @@ -27,9 +27,9 @@ jobs: - name: Install dependencies and package run: uv sync --all-extras working-directory: cli - # - name: Run tests - # run: uv run pytest - # working-directory: cli + # - name: Run tests + # run: uv run pytest + # working-directory: cli check-format: name: Check formatting runs-on: ubuntu-latest @@ -68,4 +68,4 @@ jobs: working-directory: cli - name: Check types with mypy run: uv run mypy . - working-directory: cli + working-directory: cli \ No newline at end of file diff --git a/.github/workflows/datasets-mirror-priority-1.yml b/.github/workflows/datasets-mirror-priority-1.yml index f5008b59aa..4611ec00df 100644 --- a/.github/workflows/datasets-mirror-priority-1.yml +++ b/.github/workflows/datasets-mirror-priority-1.yml @@ -1,6 +1,6 @@ name: Mirror NCBI Datasets (Priority 1 - Every 2 Hours) on: - workflow_dispatch: # Allows manual triggering to re-run all priority-1 taxa + workflow_dispatch: # Allows manual triggering to re-run all priority-1 taxa schedule: - cron: '12 */2 * * *' # Runs every two hours at 12 minutes past the hour jobs: @@ -8,15 +8,15 @@ jobs: strategy: matrix: taxon_id: - - 10244 # Mpox - - 1868215 # Orthopneumovirus - - 3048148 # Metapneumovirus hominis - - 3048448 # West nile virus - - 3052345 # Morbillivirus hominis - - 3052460 # Orthoebolavirus sudanense - - 3052462 # Orthoebolavirus zairense - - 3052505 # Orthomarburgvirus marburgense - - 3052518 # CCHFV + - 10244 # Mpox + - 1868215 # Orthopneumovirus + - 3048148 # Metapneumovirus hominis + - 3048448 # West nile virus + - 3052345 # Morbillivirus hominis + - 3052460 # Orthoebolavirus sudanense + - 3052462 # Orthoebolavirus zairense + - 3052505 # Orthomarburgvirus marburgense + - 3052518 # CCHFV fail-fast: false uses: ./.github/workflows/datasets-mirror.yml secrets: inherit diff --git a/.github/workflows/datasets-mirror-priority-2.yml b/.github/workflows/datasets-mirror-priority-2.yml index 222051754f..252afaf6fb 100644 --- a/.github/workflows/datasets-mirror-priority-2.yml +++ b/.github/workflows/datasets-mirror-priority-2.yml @@ -1,6 +1,6 @@ name: Mirror NCBI Datasets (Priority 2 - Daily) on: - workflow_dispatch: # Allows manual triggering to re-run all priority-2 taxa + workflow_dispatch: # Allows manual triggering to re-run all priority-2 taxa schedule: - cron: '12 3 * * *' # Runs daily at 03:12 UTC jobs: @@ -8,35 +8,35 @@ jobs: strategy: matrix: taxon_id: - - 11053 # Dengue type 1 - - 11060 # Dengue type 2 - - 11069 # Dengue type 3 - - 11070 # Dengue type 4 - - 11137 # Coronavirus 229E - - 11234 # Measles morbillivirus - - 11250 # RSV - - 11520 # Influenza B - - 11676 # HIV1 - - 12059 # Enterovirus - - 12637 # Dengue (all) - - 31631 # Coronavirus OC43 - - 31704 # Coxsackievirus A16 - - 39054 # Enterovirus A71 - - 42769 # Coxsackievirus A10 - - 42789 # Enterovirus D68 - - 86107 # Coxsackievirus A6 - - 118655 # Oropouche - - 138948 # Enterovirus A - - 138949 # Enterovirus B - - 138950 # Enterovirus C - - 138951 # Enterovirus D - - 162145 # Human metapneumovirus - - 197911 # Influenza A - - 208893 # RSV A - - 208895 # RSV B - - 277944 # Coronavirus NL63 - - 290028 # Coronavirus HKU1 - - 3050294 # Chickenpox virus + - 11053 # Dengue type 1 + - 11060 # Dengue type 2 + - 11069 # Dengue type 3 + - 11070 # Dengue type 4 + - 11137 # Coronavirus 229E + - 11234 # Measles morbillivirus + - 11250 # RSV + - 11520 # Influenza B + - 11676 # HIV1 + - 12059 # Enterovirus + - 12637 # Dengue (all) + - 31631 # Coronavirus OC43 + - 31704 # Coxsackievirus A16 + - 39054 # Enterovirus A71 + - 42769 # Coxsackievirus A10 + - 42789 # Enterovirus D68 + - 86107 # Coxsackievirus A6 + - 118655 # Oropouche + - 138948 # Enterovirus A + - 138949 # Enterovirus B + - 138950 # Enterovirus C + - 138951 # Enterovirus D + - 162145 # Human metapneumovirus + - 197911 # Influenza A + - 208893 # RSV A + - 208895 # RSV B + - 277944 # Coronavirus NL63 + - 290028 # Coronavirus HKU1 + - 3050294 # Chickenpox virus fail-fast: false uses: ./.github/workflows/datasets-mirror.yml secrets: inherit diff --git a/.github/workflows/datasets-mirror.yml b/.github/workflows/datasets-mirror.yml index 7aca667b43..c37d6f2e05 100644 --- a/.github/workflows/datasets-mirror.yml +++ b/.github/workflows/datasets-mirror.yml @@ -58,7 +58,7 @@ jobs: - name: Send Slack Notification env: SLACK_HOOK: ${{ secrets.SLACK_HOOK }} - run: |- + run: | curl -X POST -H 'Content-type: application/json' --data '{ "text": "๐Ÿšจ Failed to mirror NCBI datasets: ${{ github.repository }}\n๐Ÿ”— <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Details>" }' $SLACK_HOOK diff --git a/.github/workflows/docs-image.yml b/.github/workflows/docs-image.yml index cd29a78e7b..d188316aef 100644 --- a/.github/workflows/docs-image.yml +++ b/.github/workflows/docs-image.yml @@ -89,7 +89,7 @@ jobs: build-args: NODE_VERSION=${{ env.NODE_VERSION }} - name: Retag and push existing image if cache hit if: env.CACHE_HIT == 'true' - run: |- + run: | TAGS=(${{ steps.dockerMetadata.outputs.tags }}) for TAG in "${TAGS[@]}"; do docker buildx imagetools create --tag $TAG ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} diff --git a/.github/workflows/ena-submission-image.yaml b/.github/workflows/ena-submission-image.yaml index 1696f1c606..f3cba51e95 100644 --- a/.github/workflows/ena-submission-image.yaml +++ b/.github/workflows/ena-submission-image.yaml @@ -100,7 +100,7 @@ jobs: done # Run smoke test to ensure the image is functional - name: Run smoke test (unit tests in docker) - run: |- + run: | docker run \ --rm ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} \ pytest scripts/test_ena_submission.py diff --git a/.github/workflows/ena-submission-unit-tests.yaml b/.github/workflows/ena-submission-unit-tests.yaml index 3e8132ef82..70cebce98f 100644 --- a/.github/workflows/ena-submission-unit-tests.yaml +++ b/.github/workflows/ena-submission-unit-tests.yaml @@ -58,9 +58,9 @@ jobs: cache-environment: true - name: Install package run: | - uv pip install --system . + uv pip install --system . - name: Run tests - run: |- + run: | # The integration tests are run in a separate workflow # They require postgres to be running pytest \ diff --git a/.github/workflows/ena-submission-workflow-tests.yml b/.github/workflows/ena-submission-workflow-tests.yml index 0e89b97b67..f039ae5717 100644 --- a/.github/workflows/ena-submission-workflow-tests.yml +++ b/.github/workflows/ena-submission-workflow-tests.yml @@ -25,7 +25,10 @@ jobs: ports: - 5432:5432 options: >- - --health-cmd "pg_isready -U postgres" --health-interval 10s --health-timeout 5s --health-retries 5 + --health-cmd "pg_isready -U postgres" + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@v6 - name: Extract Flyway version @@ -43,7 +46,7 @@ jobs: sudo mv flyway-$FLYWAY_VERSION /opt/flyway echo "/opt/flyway" >> $GITHUB_PATH - name: Run Flyway - run: | + run: | flyway -url=jdbc:postgresql://localhost:5432/loculus \ -schemas=ena_deposition_schema \ -user=postgres \ @@ -52,19 +55,19 @@ jobs: migrate - name: Set up micromamba uses: mamba-org/setup-micromamba@v2 - with: - environment-file: ena-submission/environment.yml - micromamba-version: 'latest' - cache-environment: true + with: + environment-file: ena-submission/environment.yml + micromamba-version: 'latest' + cache-environment: true - name: Install package run: | - uv pip install --system . + uv pip install --system . shell: micromamba-shell {0} working-directory: ena-submission - name: Run tests run: | - # The unit tests are run in ena-submission-unit-tests.yaml - pytest \ - --ignore=./scripts/test_ena_submission.py + # The unit tests are run in ena-submission-unit-tests.yaml + pytest \ + --ignore=./scripts/test_ena_submission.py shell: micromamba-shell {0} working-directory: ena-submission diff --git a/.github/workflows/flyway-check.yml b/.github/workflows/flyway-check.yml index 6b50fc2c6c..2804493412 100644 --- a/.github/workflows/flyway-check.yml +++ b/.github/workflows/flyway-check.yml @@ -2,38 +2,38 @@ name: Flyway Migration Check on: pull_request: - branches: [main] + branches: [ main ] paths: - 'backend/src/main/resources/db/migration/**' jobs: check-migrations: runs-on: ubuntu-latest - + steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Check for modified migrations - run: |- - git fetch origin main:main - modified_files=$(git diff --name-only main...HEAD -- src/main/resources/db/migration/) - new_files=$(git diff --name-only --diff-filter=A main...HEAD -- src/main/resources/db/migration/) - - if [ -n "$modified_files" ]; then - echo "The following existing migration files have been modified:" - echo "$modified_files" - echo "Error: Modifying existing migrations is not allowed." - exit 1 - fi - - if [ -n "$new_files" ]; then - echo "The following new migration files have been added:" - echo "$new_files" - echo "New migrations are allowed." - fi + - uses: actions/checkout@v6 + with: + fetch-depth: 0 - if [ -z "$modified_files" ] && [ -z "$new_files" ]; then - echo "No changes to migration files detected." - fi + - name: Check for modified migrations + run: | + git fetch origin main:main + modified_files=$(git diff --name-only main...HEAD -- src/main/resources/db/migration/) + new_files=$(git diff --name-only --diff-filter=A main...HEAD -- src/main/resources/db/migration/) + + if [ -n "$modified_files" ]; then + echo "The following existing migration files have been modified:" + echo "$modified_files" + echo "Error: Modifying existing migrations is not allowed." + exit 1 + fi + + if [ -n "$new_files" ]; then + echo "The following new migration files have been added:" + echo "$new_files" + echo "New migrations are allowed." + fi + + if [ -z "$modified_files" ] && [ -z "$new_files" ]; then + echo "No changes to migration files detected." + fi diff --git a/.github/workflows/format-backend-on-request.yml b/.github/workflows/format-backend-on-request.yml index 1c1a518031..22bab866da 100644 --- a/.github/workflows/format-backend-on-request.yml +++ b/.github/workflows/format-backend-on-request.yml @@ -17,22 +17,22 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ github.head_ref }} - + - name: Set up JDK uses: actions/setup-java@v5 with: java-version: "21" distribution: "adopt" - + - name: Setup Gradle uses: gradle/actions/setup-gradle@v5 - + - name: Format Backend Code run: ./gradlew ktlintFormat working-directory: ./backend - + - name: Commit and Push Changes - run: |- + run: | git config --global user.name 'Loculus bot' git config --global user.email 'bot@loculus.org' git add -A diff --git a/.github/workflows/format-ena-deposition-on-dispatch.yml b/.github/workflows/format-ena-deposition-on-dispatch.yml index d39d8a180c..69b922349e 100644 --- a/.github/workflows/format-ena-deposition-on-dispatch.yml +++ b/.github/workflows/format-ena-deposition-on-dispatch.yml @@ -34,7 +34,7 @@ jobs: shell: micromamba-shell {0} working-directory: ena-submission - name: Commit and Push Changes - run: |- + run: | git config --global user.name 'Loculus bot' git config --global user.email 'bot@loculus.org' git add -A diff --git a/.github/workflows/format-integration-tests.yml b/.github/workflows/format-integration-tests.yml index 05bc3ca338..86c61a6abf 100644 --- a/.github/workflows/format-integration-tests.yml +++ b/.github/workflows/format-integration-tests.yml @@ -11,23 +11,23 @@ on: jobs: format: runs-on: ubuntu-latest - + defaults: run: working-directory: ./integration-tests - + steps: - uses: actions/checkout@v6 - + - name: Setup Node.js uses: actions/setup-node@v6 with: node-version: '20' cache: 'npm' cache-dependency-path: integration-tests/package-lock.json - + - name: Install dependencies run: npm ci - + - name: Check formatting - run: npm run format:check + run: npm run format:check \ No newline at end of file diff --git a/.github/workflows/format-website-on-dispatch.yml b/.github/workflows/format-website-on-dispatch.yml index 7e9d9d97ac..d23e7978ee 100644 --- a/.github/workflows/format-website-on-dispatch.yml +++ b/.github/workflows/format-website-on-dispatch.yml @@ -33,7 +33,7 @@ jobs: npm ci npm run format - name: Commit and Push Changes - run: |- + run: | git config --global user.name 'Loculus bot' git config --global user.email 'bot@loculus.org' git add -A diff --git a/.github/workflows/helm-schema-lint.yaml b/.github/workflows/helm-schema-lint.yaml index b8d6e86f1a..b3aeb47e2e 100644 --- a/.github/workflows/helm-schema-lint.yaml +++ b/.github/workflows/helm-schema-lint.yaml @@ -23,7 +23,7 @@ jobs: version: v3.18.3 - name: Run Helm lint on values files - run: |- + run: | helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml -f kubernetes/loculus/values_e2e_and_dev.yaml helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml -f kubernetes/loculus/values_preview_server.yaml diff --git a/.github/workflows/ingest-image.yml b/.github/workflows/ingest-image.yml index 6ad55d79c0..055c25c5d1 100644 --- a/.github/workflows/ingest-image.yml +++ b/.github/workflows/ingest-image.yml @@ -80,7 +80,7 @@ jobs: platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} - name: Retag and push existing image if cache hit if: env.CACHE_HIT == 'true' - run: |- + run: | TAGS=(${{ steps.dockerMetadata.outputs.tags }}) for TAG in "${TAGS[@]}"; do docker buildx imagetools create --tag $TAG ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml index 080cd9a07c..c629940fe6 100644 --- a/.github/workflows/issue-triage.yml +++ b/.github/workflows/issue-triage.yml @@ -168,5 +168,5 @@ jobs: - Be conservative when suggesting related issues - Do not duplicate information already in the PR description - claude_args: |- + claude_args: | --allowedTools "Bash(gh pr:*),Bash(gh issue list:*),Bash(gh label list)" diff --git a/.github/workflows/keycloakify-image.yml b/.github/workflows/keycloakify-image.yml index b6db401d8f..018de7ca67 100644 --- a/.github/workflows/keycloakify-image.yml +++ b/.github/workflows/keycloakify-image.yml @@ -87,7 +87,7 @@ jobs: build-args: NODE_VERSION=${{ steps.get-node-version.outputs.NODE_VERSION }} - name: Tag and push image if cache hit if: env.CACHE_HIT == 'true' - run: |- + run: | TAGS=(${{ steps.dockerMetadata.outputs.tags }}) for TAG in "${TAGS[@]}"; do docker buildx imagetools create --tag $TAG ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} diff --git a/.github/workflows/preprocessing-dummy-image.yml b/.github/workflows/preprocessing-dummy-image.yml index 204ed790ab..781deabe1a 100644 --- a/.github/workflows/preprocessing-dummy-image.yml +++ b/.github/workflows/preprocessing-dummy-image.yml @@ -79,7 +79,7 @@ jobs: platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} - name: Tag and push image if cache hit if: env.CACHE_HIT == 'true' - run: |- + run: | TAGS=(${{ steps.dockerMetadata.outputs.tags }}) for TAG in "${TAGS[@]}"; do docker buildx imagetools create --tag $TAG ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} diff --git a/.github/workflows/preprocessing-nextclade-image.yml b/.github/workflows/preprocessing-nextclade-image.yml index a4fd8c5207..98f9d7beba 100644 --- a/.github/workflows/preprocessing-nextclade-image.yml +++ b/.github/workflows/preprocessing-nextclade-image.yml @@ -80,7 +80,7 @@ jobs: platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} - name: Retag and push existing image if cache hit if: env.CACHE_HIT == 'true' - run: |- + run: | TAGS=(${{ steps.dockerMetadata.outputs.tags }}) for TAG in "${TAGS[@]}"; do docker buildx imagetools create --tag $TAG ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} diff --git a/.github/workflows/schema-dump.yml b/.github/workflows/schema-dump.yml index 642607b861..a49e61fbbb 100644 --- a/.github/workflows/schema-dump.yml +++ b/.github/workflows/schema-dump.yml @@ -11,7 +11,7 @@ on: jobs: generate-schema: runs-on: ubuntu-latest - + # Explicitly set permissions needed for committing permissions: contents: write @@ -25,59 +25,69 @@ jobs: POSTGRES_DB: postgres POSTGRES_USER: postgres options: >- - --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 ports: - 5432:5432 steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - ref: ${{ github.head_ref }} - fetch-depth: 0 - - - name: Run Flyway migrations - uses: docker://flyway/flyway:10-alpine - env: - FLYWAY_URL: jdbc:postgresql://postgres:5432/postgres - FLYWAY_USER: postgres - FLYWAY_PASSWORD: postgres - with: - args: -locations=filesystem:./backend/src/main/resources/db/migration/ migrate + - name: Checkout repository + uses: actions/checkout@v6 + with: + ref: ${{ github.head_ref }} + fetch-depth: 0 - - name: Generate new schema - uses: docker://postgres:16 - with: - args: > - bash -c "PGPASSWORD=postgres pg_dump -h postgres -U postgres -d postgres --schema-only --no-security-labels --restrict-key='dummy' > /github/workspace/backend/docs/db/schema.sql" - - name: Cat schema - run: cat backend/docs/db/schema.sql + - name: Run Flyway migrations + uses: docker://flyway/flyway:10-alpine + env: + FLYWAY_URL: jdbc:postgresql://postgres:5432/postgres + FLYWAY_USER: postgres + FLYWAY_PASSWORD: postgres + with: + args: -locations=filesystem:./backend/src/main/resources/db/migration/ migrate - - name: Stage schema file - run: git add backend/docs/db/schema.sql + - name: Generate new schema + uses: docker://postgres:16 + with: + args: > + bash -c "PGPASSWORD=postgres pg_dump + -h postgres + -U postgres + -d postgres + --schema-only + --no-security-labels + --restrict-key='dummy' + > /github/workspace/backend/docs/db/schema.sql" + - name: Cat schema + run: cat backend/docs/db/schema.sql + + - name: Stage schema file + run: git add backend/docs/db/schema.sql - - name: Check for schema changes - id: check-changes - run: | - if ! git diff --cached --quiet backend/docs/db/schema.sql; then - echo "Schema changes detected" - echo "changed=true" >> $GITHUB_OUTPUT - git diff --cached backend/docs/db/schema.sql - else - echo "No schema changes detected" - echo "changed=false" >> $GITHUB_OUTPUT - fi + - name: Check for schema changes + id: check-changes + run: | + if ! git diff --cached --quiet backend/docs/db/schema.sql; then + echo "Schema changes detected" + echo "changed=true" >> $GITHUB_OUTPUT + git diff --cached backend/docs/db/schema.sql + else + echo "No schema changes detected" + echo "changed=false" >> $GITHUB_OUTPUT + fi - - name: Handle schema changes - if: steps.check-changes.outputs.changed == 'true' - run: |- - if [[ "${{ contains(github.event.pull_request.labels.*.name, 'update_db_schema') }}" == "true" ]]; then - git config --global user.name 'GitHub Action' - git config --global user.email 'action@github.com' - git commit -m "Update schema documentation based on migration changes" - git push - else - echo "::error::Schema changes detected but 'update-schema' label is not present on the PR" - echo "Please add the 'update_db_schema' label if these changes are intentional" - exit 1 - fi + - name: Handle schema changes + if: steps.check-changes.outputs.changed == 'true' + run: | + if [[ "${{ contains(github.event.pull_request.labels.*.name, 'update_db_schema') }}" == "true" ]]; then + git config --global user.name 'GitHub Action' + git config --global user.email 'action@github.com' + git commit -m "Update schema documentation based on migration changes" + git push + else + echo "::error::Schema changes detected but 'update-schema' label is not present on the PR" + echo "Please add the 'update_db_schema' label if these changes are intentional" + exit 1 + fi diff --git a/.github/workflows/update-argocd-metadata.yml b/.github/workflows/update-argocd-metadata.yml index 2fdc7c797e..5b44ea9914 100644 --- a/.github/workflows/update-argocd-metadata.yml +++ b/.github/workflows/update-argocd-metadata.yml @@ -106,7 +106,7 @@ jobs: "head_short_sha_7": "${{ steps.get_sha.outputs.sha }}" }' > config.json - name: Commit and Push Changes - run: |- + run: | git config --global user.name 'Loculus bot' git config --global user.email 'bot@loculus.org' git add config.json diff --git a/.github/workflows/website-image.yml b/.github/workflows/website-image.yml index 0800fcdae0..8d45537275 100644 --- a/.github/workflows/website-image.yml +++ b/.github/workflows/website-image.yml @@ -156,7 +156,7 @@ jobs: docker buildx imagetools inspect ${{ env.DOCKER_IMAGE_NAME }}:${{ needs.check-cache.outputs.dir-hash }} - name: Retag and push existing image if: needs.check-cache.outputs.cache-hit == 'true' - run: |- + run: | METADATA='${{ steps.dockerMetadata.outputs.json }}' TAGS=$(jq -r '.tags[]' <<< "$METADATA") for TAG in $TAGS; do diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index da1b4ed811..df4e073db1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,36 +1,36 @@ repos: - - repo: https://github.com/astral-sh/ruff-pre-commit - # Ruff version. - rev: v0.14.2 - hooks: - # Run the linter and fix issues. - - id: ruff-check - args: [--fix] - files: ^ena-submission/ - # Run lint but only fix auto-fixable issues. - - id: ruff-check - args: [--fix-only] - files: ^preprocessing/nextclade/ - # Run the formatter. - - id: ruff-format - files: ^ena-submission/|^preprocessing/nextclade/ - - repo: local - hooks: - - id: prettier-values-schema - name: prettier (values.schema.json) - entry: npx prettier@3.6.2 --write - language: system - files: ^kubernetes/loculus/values\.schema\.json$ - - id: helm-lint - name: helm lint - entry: bash - args: - - -c - - | - set -euo pipefail - helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml - helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml -f kubernetes/loculus/values_e2e_and_dev.yaml - helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml -f kubernetes/loculus/values_preview_server.yaml - language: system - pass_filenames: false - files: ^kubernetes/loculus/ +- repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.14.2 + hooks: + # Run the linter and fix issues. + - id: ruff-check + args: [ --fix ] + files: ^ena-submission/ + # Run lint but only fix auto-fixable issues. + - id: ruff-check + args: [ --fix-only ] + files: ^preprocessing/nextclade/ + # Run the formatter. + - id: ruff-format + files: ^ena-submission/|^preprocessing/nextclade/ +- repo: local + hooks: + - id: prettier-values-schema + name: prettier (values.schema.json) + entry: npx prettier@3.6.2 --write + language: system + files: ^kubernetes/loculus/values\.schema\.json$ + - id: helm-lint + name: helm lint + entry: bash + args: + - -c + - | + set -euo pipefail + helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml + helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml -f kubernetes/loculus/values_e2e_and_dev.yaml + helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml -f kubernetes/loculus/values_preview_server.yaml + language: system + pass_filenames: false + files: ^kubernetes/loculus/ diff --git a/ena-submission/environment.yml b/ena-submission/environment.yml index 5556e4efbc..5d3c4910c4 100644 --- a/ena-submission/environment.yml +++ b/ena-submission/environment.yml @@ -5,7 +5,7 @@ channels: - nodefaults dependencies: # Core Python dependencies - - python=3.12.9 # Pinned to 3.12.x series, check for patch updates within 3.12 + - python=3.12.9 # Pinned to 3.12.x series, check for patch updates within 3.12 - pip=25.2 - uv=0.9.5 # Extra dependencies diff --git a/ena-submission/test/test_config.yaml b/ena-submission/test/test_config.yaml index a3759cc452..289f63ff2a 100644 --- a/ena-submission/test/test_config.yaml +++ b/ena-submission/test/test_config.yaml @@ -19,7 +19,7 @@ enaOrganisms: scientific_name: Orthonairovirus haemorrhagiae taxon_id: 3052518 organismName: "Crimean-Congo Hemorrhagic Fever Virus" - segments: + segments: - L - M - S @@ -92,7 +92,7 @@ enaOrganisms: scientific_name: West Nile virus taxon_id: 11082 organismName: "West Nile Virus" - segments: + segments: - main externalMetadata: - externalMetadataUpdater: ena @@ -133,4 +133,4 @@ enaOrganisms: type: int - externalMetadataUpdater: ena name: sraRunAccession - type: string + type: string \ No newline at end of file diff --git a/ingest/config/defaults.yaml b/ingest/config/defaults.yaml index 7296e01b30..308dd12396 100644 --- a/ingest/config/defaults.yaml +++ b/ingest/config/defaults.yaml @@ -62,16 +62,16 @@ ncbi_mappings: - ncbiHostCommonName - ncbiPurposeOfSampling - ncbiHostSex -group_name: insdc_ingest_group # Used only to set the group name, never read +group_name: insdc_ingest_group # Used only to set the group name, never read username: insdc_ingest_user password: insdc_ingest_user keycloak_client_id: backend-client subsample_fraction: 1.0 -approve_timeout_min: "25" # Cronjobs run every 30min, make approve stop before it is forced to stop by argocd +approve_timeout_min: "25" # Cronjobs run every 30min, make approve stop before it is forced to stop by argocd db_username: postgres db_password: unsecure db_url: "jdbc:postgresql://127.0.0.1:5432/loculus" -batch_chunk_size: 10000 # Batch size for submitting sequences to Loculus backend +batch_chunk_size: 10000 # Batch size for submitting sequences to Loculus backend nextclade_dataset_server: https://data.clades.nextstrain.org/v3 time_between_approve_requests_seconds: 60 backend_request_timeout_seconds: 600 diff --git a/ingest/environment.yml b/ingest/environment.yml index 516f2322f5..2ac91266e7 100644 --- a/ingest/environment.yml +++ b/ingest/environment.yml @@ -5,7 +5,7 @@ channels: - nodefaults dependencies: # Core Python dependencies - - python=3.12.9 # Pinned to 3.12.x series, check for patch updates within 3.12 + - python=3.12.9 # Pinned to 3.12.x series, check for patch updates within 3.12 # Extra dependencies - biopython=1.85 - click=8.3.0 diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index ac326b65d3..9b03b14a50 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -1391,8 +1391,8 @@ defaultOrganismConfig: &defaultOrganismConfig segments: - name: main references: - - name: singleReference - genes: [] + - name: singleReference + genes: [] nextclade_dataset_server: https://data.clades.nextstrain.org/v3 ingest: &ingest image: ghcr.io/loculus-project/ingest @@ -1415,9 +1415,9 @@ defaultOrganisms: segments: - name: main references: - - name: singleReference - nextclade_dataset_name: nextstrain/ebola/sudan - genes: [NP, VP35, VP40, GP, sGP, ssGP, VP30, VP24, L] + - name: singleReference + nextclade_dataset_name: nextstrain/ebola/sudan + genes: [NP, VP35, VP40, GP, sGP, ssGP, VP30, VP24, L] nextclade_dataset_server: https://raw.githubusercontent.com/nextstrain/nextclade_data/ebola/data_output taxon_id: 186540 scientific_name: "Sudan ebolavirus" @@ -1512,9 +1512,9 @@ defaultOrganisms: segments: - name: main references: - - name: singleReference - nextclade_dataset_name: nextstrain/wnv/all-lineages - genes: [capsid, prM, env, NS1, NS2A, NS2B, NS3, NS4A, 2K, NS4B, NS5] + - name: singleReference + nextclade_dataset_name: nextstrain/wnv/all-lineages + genes: [capsid, prM, env, NS1, NS2A, NS2B, NS3, NS4A, 2K, NS4B, NS5] nextclade_dataset_server: https://raw.githubusercontent.com/nextstrain/nextclade_data/wnv/data_output taxon_id: 11082 scientific_name: "West Nile virus" @@ -1611,7 +1611,7 @@ defaultOrganisms: required: true type: string lineageSystem: pangoLineage - options: + options: - name: A - name: A.1 - name: A.1.1 @@ -1723,7 +1723,7 @@ defaultOrganisms: required: true type: string lineageSystem: alternativeLineage - options: + options: - name: A - name: A.1 - name: A.1.1 @@ -1779,8 +1779,8 @@ defaultOrganisms: segments: - name: main references: - - name: singleReference - genes: [] + - name: singleReference + genes: [] referenceGenomes: - name: main references: @@ -1853,19 +1853,19 @@ defaultOrganisms: segments: - name: L references: - - nextclade_dataset_name: community/pathoplexus/cchfv/L - name: singleReference - genes: [RdRp] + - nextclade_dataset_name: community/pathoplexus/cchfv/L + name: singleReference + genes: [RdRp] - name: M references: - - nextclade_dataset_name: community/pathoplexus/cchfv/M - name: singleReference - genes: [GPC] + - nextclade_dataset_name: community/pathoplexus/cchfv/M + name: singleReference + genes: [GPC] - name: S references: - - nextclade_dataset_name: community/pathoplexus/cchfv/S - name: singleReference - genes: [NP] + - nextclade_dataset_name: community/pathoplexus/cchfv/S + name: singleReference + genes: [NP] ingest: <<: *ingest configFile: @@ -1990,32 +1990,32 @@ defaultOrganisms: segments: - name: L references: - - nextclade_dataset_name: cchfv/L - name: singleReference - genes: [RdRp] + - nextclade_dataset_name: cchfv/L + name: singleReference + genes: [RdRp] - name: M references: - - nextclade_dataset_name: cchfv/M-MH396653 - name: MH396653 - genes: [GPC] - - nextclade_dataset_name: cchfv/M-OR047158 - name: OR047158 - genes: [GPC] + - nextclade_dataset_name: cchfv/M-MH396653 + name: MH396653 + genes: [GPC] + - nextclade_dataset_name: cchfv/M-OR047158 + name: OR047158 + genes: [GPC] - name: S references: - - nextclade_dataset_name: cchfv/S-1and6 - name: 1and6 - genes: [NP] - - nextclade_dataset_name: cchfv/S-2to5 - name: 2to5 - genes: [NP] + - nextclade_dataset_name: cchfv/S-1and6 + name: 1and6 + genes: [NP] + - nextclade_dataset_name: cchfv/S-2to5 + name: 2to5 + genes: [NP] ingest: <<: *ingest configFile: taxon_id: 3052518 segment_identification: method: "align" - nextclade_dataset_name: community/pathoplexus/cchfv #TODO: allow multiple ref in ingest grouping algo + nextclade_dataset_name: community/pathoplexus/cchfv #TODO: allow multiple ref in ingest grouping algo enaDeposition: singleReference: configFile: @@ -2164,23 +2164,23 @@ defaultOrganisms: segments: - name: main references: - - name: CV-A16 - nextclade_dataset_name: enpen/enterovirus/cv-a16 - accepted_dataset_matches: ["community/hodcroftlab/enterovirus/cva16", "community/hodcroftlab/enterovirus/enterovirus/linked/CV-A16"] - genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] - - name: CV-A10 - nextclade_dataset_name: enpen/enterovirus/cv-a10 - accepted_dataset_matches: ["community/hodcroftlab/enterovirus/enterovirus/linked/CV-A10"] - genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] - - name: EV-A71 - nextclade_dataset_name: enpen/enterovirus/ev-a71 - accepted_dataset_matches: ["community/hodcroftlab/enterovirus/enterovirus/linked/EV-A71"] - genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] - - name: EV-D68 - accepted_dataset_matches: ["community/hodcroftlab/enterovirus/enterovirus/linked/EV-D68"] - genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] - nextclade_dataset_name: enpen/enterovirus/ev-d68 - nextclade_dataset_server: https://data.clades.nextstrain.org/v3 + - name: CV-A16 + nextclade_dataset_name: enpen/enterovirus/cv-a16 + accepted_dataset_matches: ["community/hodcroftlab/enterovirus/cva16", "community/hodcroftlab/enterovirus/enterovirus/linked/CV-A16"] + genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] + - name: CV-A10 + nextclade_dataset_name: enpen/enterovirus/cv-a10 + accepted_dataset_matches: ["community/hodcroftlab/enterovirus/enterovirus/linked/CV-A10"] + genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] + - name: EV-A71 + nextclade_dataset_name: enpen/enterovirus/ev-a71 + accepted_dataset_matches: ["community/hodcroftlab/enterovirus/enterovirus/linked/EV-A71"] + genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] + - name: EV-D68 + accepted_dataset_matches: ["community/hodcroftlab/enterovirus/enterovirus/linked/EV-D68"] + genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] + nextclade_dataset_name: enpen/enterovirus/ev-d68 + nextclade_dataset_server: https://data.clades.nextstrain.org/v3 nextclade_dataset_server: https://raw.githubusercontent.com/nextstrain/nextclade_data/evs-datasets/data_output ingest: <<: *ingest @@ -2418,7 +2418,7 @@ enforceHTTPS: true registrationTermsMessage: > You must agree to the terms of use. -enaDeposition: +enaDeposition: submitToEnaProduction: false enaDbName: Loculus enaUniqueSuffix: Loculus diff --git a/kubernetes/loculus/values_e2e_and_dev.yaml b/kubernetes/loculus/values_e2e_and_dev.yaml index 032af7efee..f7fce7b84d 100644 --- a/kubernetes/loculus/values_e2e_and_dev.yaml +++ b/kubernetes/loculus/values_e2e_and_dev.yaml @@ -14,4 +14,4 @@ auth: verifyEmail: false host: localhost:3000 siloImport: - pollIntervalSeconds: 5 + pollIntervalSeconds: 5 \ No newline at end of file diff --git a/kubernetes/loculus/values_preview_server.yaml b/kubernetes/loculus/values_preview_server.yaml index c6a1ea97d2..a7acaa3523 100644 --- a/kubernetes/loculus/values_preview_server.yaml +++ b/kubernetes/loculus/values_preview_server.yaml @@ -140,4 +140,4 @@ defaultResources: cpu: "20m" limits: memory: "1Gi" - cpu: "20m" + cpu: "20m" \ No newline at end of file diff --git a/preprocessing/dummy/lineage-alternative.yaml b/preprocessing/dummy/lineage-alternative.yaml index 8f7008394d..b2b34e1fa8 100644 --- a/preprocessing/dummy/lineage-alternative.yaml +++ b/preprocessing/dummy/lineage-alternative.yaml @@ -4,30 +4,30 @@ A: A.1: aliases: [] parents: - - A + - A A.2: aliases: [] parents: - - A + - A A.1.1: aliases: [] parents: - - A.1 - - A.2 + - A.1 + - A.2 B: aliases: [] parents: [] B.1: aliases: [] parents: - - B + - B B.1.1: aliases: - - C + - C parents: - - B.1 + - B.1 C.1: aliases: - - B.1.1.1 + - B.1.1.1 parents: - - C + - C diff --git a/preprocessing/dummy/lineage.yaml b/preprocessing/dummy/lineage.yaml index 52236c2360..0c533e1a6c 100644 --- a/preprocessing/dummy/lineage.yaml +++ b/preprocessing/dummy/lineage.yaml @@ -4,13 +4,13 @@ A: A.1: aliases: [] parents: - - A + - A A.1.1: aliases: - - B + - B parents: - - A.1 + - A.1 A.2: aliases: [] parents: - - A + - A diff --git a/preprocessing/nextclade/tests/embl_required_metadata.yaml b/preprocessing/nextclade/tests/embl_required_metadata.yaml index 18641f04a9..919e8f94e9 100644 --- a/preprocessing/nextclade/tests/embl_required_metadata.yaml +++ b/preprocessing/nextclade/tests/embl_required_metadata.yaml @@ -20,4 +20,4 @@ processing_spec: type: date function: parse_and_assert_past_date inputs: - date: sampleCollectionDate + date: sampleCollectionDate \ No newline at end of file diff --git a/preprocessing/nextclade/tests/multi_reference_config.yaml b/preprocessing/nextclade/tests/multi_reference_config.yaml index f2f2aae0b9..5784f1e403 100644 --- a/preprocessing/nextclade/tests/multi_reference_config.yaml +++ b/preprocessing/nextclade/tests/multi_reference_config.yaml @@ -7,14 +7,14 @@ nextclade_dataset_server: TEST segments: - name: main references: - - name: ebola-sudan - nextclade_dataset_name: ebola-dataset/ebola-sudan - accepted_dataset_matches: [ebola-sudan] - genes: [NPEbolaSudan, VP35EbolaSudan] # Names of genes in nextclade dataset gff3 file - - name: ebola-zaire - nextclade_dataset_name: ebola-dataset/ebola-zaire - accepted_dataset_matches: [ebola-zaire] - genes: [VP24EbolaZaire, LEbolaZaire] + - name: ebola-sudan + nextclade_dataset_name: ebola-dataset/ebola-sudan + accepted_dataset_matches: [ebola-sudan] + genes: [NPEbolaSudan, VP35EbolaSudan] # Names of genes in nextclade dataset gff3 file + - name: ebola-zaire + nextclade_dataset_name: ebola-dataset/ebola-zaire + accepted_dataset_matches: [ebola-zaire] + genes: [VP24EbolaZaire, LEbolaZaire] organism: multi-ebola-test processing_spec: subtype: diff --git a/preprocessing/nextclade/tests/multi_segment_config.yaml b/preprocessing/nextclade/tests/multi_segment_config.yaml index cfded28c62..2a9fa7a5b9 100644 --- a/preprocessing/nextclade/tests/multi_segment_config.yaml +++ b/preprocessing/nextclade/tests/multi_segment_config.yaml @@ -6,12 +6,12 @@ segment_classification_method: "minimizer" segments: - name: ebola-sudan references: - - nextclade_dataset_name: ebola-dataset/ebola-sudan - genes: [NPEbolaSudan, VP35EbolaSudan] + - nextclade_dataset_name: ebola-dataset/ebola-sudan + genes: [NPEbolaSudan, VP35EbolaSudan] - name: ebola-zaire references: - - nextclade_dataset_name: ebola-dataset/ebola-zaire - genes: [VP24EbolaZaire, LEbolaZaire] + - nextclade_dataset_name: ebola-dataset/ebola-zaire + genes: [VP24EbolaZaire, LEbolaZaire] organism: multi-ebola-test processing_spec: totalInsertedNucs_ebola-zaire: diff --git a/preprocessing/nextclade/tests/multi_segment_multi_reference.yaml b/preprocessing/nextclade/tests/multi_segment_multi_reference.yaml index b5f69f9702..fea2a6e783 100644 --- a/preprocessing/nextclade/tests/multi_segment_multi_reference.yaml +++ b/preprocessing/nextclade/tests/multi_segment_multi_reference.yaml @@ -8,20 +8,20 @@ diamond_dmnd_url: FAKE_URL segments: - name: L references: - - name: singleReference - nextclade_dataset_name: CCHF/L - accepted_dataset_matches: [L] - genes: [RdRp] + - name: singleReference + nextclade_dataset_name: CCHF/L + accepted_dataset_matches: [L] + genes: [RdRp] - name: S references: - - name: 1and6 - nextclade_dataset_name: CCHF/S-1and6 - accepted_dataset_matches: [S-1and6] - genes: [NP] - - name: 2to5 - nextclade_dataset_name: CCHF/S-2to5 - accepted_dataset_matches: [S-2to5] - genes: [NP] + - name: 1and6 + nextclade_dataset_name: CCHF/S-1and6 + accepted_dataset_matches: [S-1and6] + genes: [NP] + - name: 2to5 + nextclade_dataset_name: CCHF/S-2to5 + accepted_dataset_matches: [S-2to5] + genes: [NP] organism: multi-reference-cchf-test processing_spec: subtype_S: diff --git a/preprocessing/nextclade/tests/no_alignment_config.yaml b/preprocessing/nextclade/tests/no_alignment_config.yaml index da2eb5c58c..569a649a5f 100644 --- a/preprocessing/nextclade/tests/no_alignment_config.yaml +++ b/preprocessing/nextclade/tests/no_alignment_config.yaml @@ -87,7 +87,7 @@ processing_spec: extracted_regex_field: function: extract_regex args: - pattern: "^EPI_ISL_(?P[0-9]+)?$" # Note the optional group to allow for testing of missing values + pattern: "^EPI_ISL_(?P[0-9]+)?$" # Note the optional group to allow for testing of missing values capture_group: id inputs: regex_field: regex_field diff --git a/preprocessing/nextclade/tests/single_segment_config.yaml b/preprocessing/nextclade/tests/single_segment_config.yaml index 3c8f31bcc4..408f0beb41 100644 --- a/preprocessing/nextclade/tests/single_segment_config.yaml +++ b/preprocessing/nextclade/tests/single_segment_config.yaml @@ -8,8 +8,8 @@ db_name: "Loculus" segments: - name: main references: - - nextclade_dataset_name: ebola-sudan - genes: [NPEbolaSudan, VP35EbolaSudan] + - nextclade_dataset_name: ebola-sudan + genes: [NPEbolaSudan, VP35EbolaSudan] organism: ebola-sudan-test processing_spec: completeness: From 2084ae5eb756ce6ba20dc07a853b678dd5a96002 Mon Sep 17 00:00:00 2001 From: tombch Date: Wed, 4 Mar 2026 16:55:16 +0100 Subject: [PATCH 07/10] Only run on push to main branch and prs --- .github/workflows/yamlfmt-lint.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/yamlfmt-lint.yml b/.github/workflows/yamlfmt-lint.yml index a87cf039f8..23aed02536 100644 --- a/.github/workflows/yamlfmt-lint.yml +++ b/.github/workflows/yamlfmt-lint.yml @@ -4,7 +4,6 @@ on: push: branches: - main - - yamlfmt-lint #ย For testing workflow_dispatch: jobs: From 3772057eb4aa2181522bf840f2592587068b4fac Mon Sep 17 00:00:00 2001 From: tombch Date: Wed, 4 Mar 2026 17:07:39 +0100 Subject: [PATCH 08/10] Excluding workflows from formatting/linting --- .yamlfmt.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.yamlfmt.yaml b/.yamlfmt.yaml index a799fd26a5..88a0f5e6d8 100644 --- a/.yamlfmt.yaml +++ b/.yamlfmt.yaml @@ -7,3 +7,4 @@ formatter: gitignore_excludes: true exclude: - kubernetes/loculus/templates + - .github From 547af7e7e6c539325a5917a64ea75fc536f92e55 Mon Sep 17 00:00:00 2001 From: tombch Date: Wed, 4 Mar 2026 17:09:19 +0100 Subject: [PATCH 09/10] Reformatted other yaml files --- .pre-commit-config.yaml | 70 +++++------ ena-submission/environment.yml | 2 +- ena-submission/test/test_config.yaml | 6 +- ingest/config/defaults.yaml | 6 +- ingest/environment.yml | 2 +- kubernetes/loculus/values.yaml | 110 +++++++++--------- kubernetes/loculus/values_e2e_and_dev.yaml | 2 +- kubernetes/loculus/values_preview_server.yaml | 2 +- preprocessing/dummy/lineage-alternative.yaml | 18 +-- preprocessing/dummy/lineage.yaml | 8 +- .../tests/embl_required_metadata.yaml | 2 +- .../tests/multi_reference_config.yaml | 16 +-- .../nextclade/tests/multi_segment_config.yaml | 8 +- .../tests/multi_segment_multi_reference.yaml | 24 ++-- .../nextclade/tests/no_alignment_config.yaml | 2 +- .../tests/single_segment_config.yaml | 4 +- 16 files changed, 141 insertions(+), 141 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df4e073db1..da1b4ed811 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,36 +1,36 @@ repos: -- repo: https://github.com/astral-sh/ruff-pre-commit - # Ruff version. - rev: v0.14.2 - hooks: - # Run the linter and fix issues. - - id: ruff-check - args: [ --fix ] - files: ^ena-submission/ - # Run lint but only fix auto-fixable issues. - - id: ruff-check - args: [ --fix-only ] - files: ^preprocessing/nextclade/ - # Run the formatter. - - id: ruff-format - files: ^ena-submission/|^preprocessing/nextclade/ -- repo: local - hooks: - - id: prettier-values-schema - name: prettier (values.schema.json) - entry: npx prettier@3.6.2 --write - language: system - files: ^kubernetes/loculus/values\.schema\.json$ - - id: helm-lint - name: helm lint - entry: bash - args: - - -c - - | - set -euo pipefail - helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml - helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml -f kubernetes/loculus/values_e2e_and_dev.yaml - helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml -f kubernetes/loculus/values_preview_server.yaml - language: system - pass_filenames: false - files: ^kubernetes/loculus/ + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.14.2 + hooks: + # Run the linter and fix issues. + - id: ruff-check + args: [--fix] + files: ^ena-submission/ + # Run lint but only fix auto-fixable issues. + - id: ruff-check + args: [--fix-only] + files: ^preprocessing/nextclade/ + # Run the formatter. + - id: ruff-format + files: ^ena-submission/|^preprocessing/nextclade/ + - repo: local + hooks: + - id: prettier-values-schema + name: prettier (values.schema.json) + entry: npx prettier@3.6.2 --write + language: system + files: ^kubernetes/loculus/values\.schema\.json$ + - id: helm-lint + name: helm lint + entry: bash + args: + - -c + - | + set -euo pipefail + helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml + helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml -f kubernetes/loculus/values_e2e_and_dev.yaml + helm lint kubernetes/loculus -f kubernetes/loculus/values.yaml -f kubernetes/loculus/values_preview_server.yaml + language: system + pass_filenames: false + files: ^kubernetes/loculus/ diff --git a/ena-submission/environment.yml b/ena-submission/environment.yml index 5d3c4910c4..5556e4efbc 100644 --- a/ena-submission/environment.yml +++ b/ena-submission/environment.yml @@ -5,7 +5,7 @@ channels: - nodefaults dependencies: # Core Python dependencies - - python=3.12.9 # Pinned to 3.12.x series, check for patch updates within 3.12 + - python=3.12.9 # Pinned to 3.12.x series, check for patch updates within 3.12 - pip=25.2 - uv=0.9.5 # Extra dependencies diff --git a/ena-submission/test/test_config.yaml b/ena-submission/test/test_config.yaml index 289f63ff2a..a3759cc452 100644 --- a/ena-submission/test/test_config.yaml +++ b/ena-submission/test/test_config.yaml @@ -19,7 +19,7 @@ enaOrganisms: scientific_name: Orthonairovirus haemorrhagiae taxon_id: 3052518 organismName: "Crimean-Congo Hemorrhagic Fever Virus" - segments: + segments: - L - M - S @@ -92,7 +92,7 @@ enaOrganisms: scientific_name: West Nile virus taxon_id: 11082 organismName: "West Nile Virus" - segments: + segments: - main externalMetadata: - externalMetadataUpdater: ena @@ -133,4 +133,4 @@ enaOrganisms: type: int - externalMetadataUpdater: ena name: sraRunAccession - type: string \ No newline at end of file + type: string diff --git a/ingest/config/defaults.yaml b/ingest/config/defaults.yaml index 308dd12396..7296e01b30 100644 --- a/ingest/config/defaults.yaml +++ b/ingest/config/defaults.yaml @@ -62,16 +62,16 @@ ncbi_mappings: - ncbiHostCommonName - ncbiPurposeOfSampling - ncbiHostSex -group_name: insdc_ingest_group # Used only to set the group name, never read +group_name: insdc_ingest_group # Used only to set the group name, never read username: insdc_ingest_user password: insdc_ingest_user keycloak_client_id: backend-client subsample_fraction: 1.0 -approve_timeout_min: "25" # Cronjobs run every 30min, make approve stop before it is forced to stop by argocd +approve_timeout_min: "25" # Cronjobs run every 30min, make approve stop before it is forced to stop by argocd db_username: postgres db_password: unsecure db_url: "jdbc:postgresql://127.0.0.1:5432/loculus" -batch_chunk_size: 10000 # Batch size for submitting sequences to Loculus backend +batch_chunk_size: 10000 # Batch size for submitting sequences to Loculus backend nextclade_dataset_server: https://data.clades.nextstrain.org/v3 time_between_approve_requests_seconds: 60 backend_request_timeout_seconds: 600 diff --git a/ingest/environment.yml b/ingest/environment.yml index 2ac91266e7..516f2322f5 100644 --- a/ingest/environment.yml +++ b/ingest/environment.yml @@ -5,7 +5,7 @@ channels: - nodefaults dependencies: # Core Python dependencies - - python=3.12.9 # Pinned to 3.12.x series, check for patch updates within 3.12 + - python=3.12.9 # Pinned to 3.12.x series, check for patch updates within 3.12 # Extra dependencies - biopython=1.85 - click=8.3.0 diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index 9b03b14a50..ac326b65d3 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -1391,8 +1391,8 @@ defaultOrganismConfig: &defaultOrganismConfig segments: - name: main references: - - name: singleReference - genes: [] + - name: singleReference + genes: [] nextclade_dataset_server: https://data.clades.nextstrain.org/v3 ingest: &ingest image: ghcr.io/loculus-project/ingest @@ -1415,9 +1415,9 @@ defaultOrganisms: segments: - name: main references: - - name: singleReference - nextclade_dataset_name: nextstrain/ebola/sudan - genes: [NP, VP35, VP40, GP, sGP, ssGP, VP30, VP24, L] + - name: singleReference + nextclade_dataset_name: nextstrain/ebola/sudan + genes: [NP, VP35, VP40, GP, sGP, ssGP, VP30, VP24, L] nextclade_dataset_server: https://raw.githubusercontent.com/nextstrain/nextclade_data/ebola/data_output taxon_id: 186540 scientific_name: "Sudan ebolavirus" @@ -1512,9 +1512,9 @@ defaultOrganisms: segments: - name: main references: - - name: singleReference - nextclade_dataset_name: nextstrain/wnv/all-lineages - genes: [capsid, prM, env, NS1, NS2A, NS2B, NS3, NS4A, 2K, NS4B, NS5] + - name: singleReference + nextclade_dataset_name: nextstrain/wnv/all-lineages + genes: [capsid, prM, env, NS1, NS2A, NS2B, NS3, NS4A, 2K, NS4B, NS5] nextclade_dataset_server: https://raw.githubusercontent.com/nextstrain/nextclade_data/wnv/data_output taxon_id: 11082 scientific_name: "West Nile virus" @@ -1611,7 +1611,7 @@ defaultOrganisms: required: true type: string lineageSystem: pangoLineage - options: + options: - name: A - name: A.1 - name: A.1.1 @@ -1723,7 +1723,7 @@ defaultOrganisms: required: true type: string lineageSystem: alternativeLineage - options: + options: - name: A - name: A.1 - name: A.1.1 @@ -1779,8 +1779,8 @@ defaultOrganisms: segments: - name: main references: - - name: singleReference - genes: [] + - name: singleReference + genes: [] referenceGenomes: - name: main references: @@ -1853,19 +1853,19 @@ defaultOrganisms: segments: - name: L references: - - nextclade_dataset_name: community/pathoplexus/cchfv/L - name: singleReference - genes: [RdRp] + - nextclade_dataset_name: community/pathoplexus/cchfv/L + name: singleReference + genes: [RdRp] - name: M references: - - nextclade_dataset_name: community/pathoplexus/cchfv/M - name: singleReference - genes: [GPC] + - nextclade_dataset_name: community/pathoplexus/cchfv/M + name: singleReference + genes: [GPC] - name: S references: - - nextclade_dataset_name: community/pathoplexus/cchfv/S - name: singleReference - genes: [NP] + - nextclade_dataset_name: community/pathoplexus/cchfv/S + name: singleReference + genes: [NP] ingest: <<: *ingest configFile: @@ -1990,32 +1990,32 @@ defaultOrganisms: segments: - name: L references: - - nextclade_dataset_name: cchfv/L - name: singleReference - genes: [RdRp] + - nextclade_dataset_name: cchfv/L + name: singleReference + genes: [RdRp] - name: M references: - - nextclade_dataset_name: cchfv/M-MH396653 - name: MH396653 - genes: [GPC] - - nextclade_dataset_name: cchfv/M-OR047158 - name: OR047158 - genes: [GPC] + - nextclade_dataset_name: cchfv/M-MH396653 + name: MH396653 + genes: [GPC] + - nextclade_dataset_name: cchfv/M-OR047158 + name: OR047158 + genes: [GPC] - name: S references: - - nextclade_dataset_name: cchfv/S-1and6 - name: 1and6 - genes: [NP] - - nextclade_dataset_name: cchfv/S-2to5 - name: 2to5 - genes: [NP] + - nextclade_dataset_name: cchfv/S-1and6 + name: 1and6 + genes: [NP] + - nextclade_dataset_name: cchfv/S-2to5 + name: 2to5 + genes: [NP] ingest: <<: *ingest configFile: taxon_id: 3052518 segment_identification: method: "align" - nextclade_dataset_name: community/pathoplexus/cchfv #TODO: allow multiple ref in ingest grouping algo + nextclade_dataset_name: community/pathoplexus/cchfv #TODO: allow multiple ref in ingest grouping algo enaDeposition: singleReference: configFile: @@ -2164,23 +2164,23 @@ defaultOrganisms: segments: - name: main references: - - name: CV-A16 - nextclade_dataset_name: enpen/enterovirus/cv-a16 - accepted_dataset_matches: ["community/hodcroftlab/enterovirus/cva16", "community/hodcroftlab/enterovirus/enterovirus/linked/CV-A16"] - genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] - - name: CV-A10 - nextclade_dataset_name: enpen/enterovirus/cv-a10 - accepted_dataset_matches: ["community/hodcroftlab/enterovirus/enterovirus/linked/CV-A10"] - genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] - - name: EV-A71 - nextclade_dataset_name: enpen/enterovirus/ev-a71 - accepted_dataset_matches: ["community/hodcroftlab/enterovirus/enterovirus/linked/EV-A71"] - genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] - - name: EV-D68 - accepted_dataset_matches: ["community/hodcroftlab/enterovirus/enterovirus/linked/EV-D68"] - genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] - nextclade_dataset_name: enpen/enterovirus/ev-d68 - nextclade_dataset_server: https://data.clades.nextstrain.org/v3 + - name: CV-A16 + nextclade_dataset_name: enpen/enterovirus/cv-a16 + accepted_dataset_matches: ["community/hodcroftlab/enterovirus/cva16", "community/hodcroftlab/enterovirus/enterovirus/linked/CV-A16"] + genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] + - name: CV-A10 + nextclade_dataset_name: enpen/enterovirus/cv-a10 + accepted_dataset_matches: ["community/hodcroftlab/enterovirus/enterovirus/linked/CV-A10"] + genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] + - name: EV-A71 + nextclade_dataset_name: enpen/enterovirus/ev-a71 + accepted_dataset_matches: ["community/hodcroftlab/enterovirus/enterovirus/linked/EV-A71"] + genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] + - name: EV-D68 + accepted_dataset_matches: ["community/hodcroftlab/enterovirus/enterovirus/linked/EV-D68"] + genes: ["VP4", "VP2", "VP3", "VP1", "2A", "2B", "2C", "3A", "3B", "3C", "3D"] + nextclade_dataset_name: enpen/enterovirus/ev-d68 + nextclade_dataset_server: https://data.clades.nextstrain.org/v3 nextclade_dataset_server: https://raw.githubusercontent.com/nextstrain/nextclade_data/evs-datasets/data_output ingest: <<: *ingest @@ -2418,7 +2418,7 @@ enforceHTTPS: true registrationTermsMessage: > You must agree to the terms of use. -enaDeposition: +enaDeposition: submitToEnaProduction: false enaDbName: Loculus enaUniqueSuffix: Loculus diff --git a/kubernetes/loculus/values_e2e_and_dev.yaml b/kubernetes/loculus/values_e2e_and_dev.yaml index f7fce7b84d..032af7efee 100644 --- a/kubernetes/loculus/values_e2e_and_dev.yaml +++ b/kubernetes/loculus/values_e2e_and_dev.yaml @@ -14,4 +14,4 @@ auth: verifyEmail: false host: localhost:3000 siloImport: - pollIntervalSeconds: 5 \ No newline at end of file + pollIntervalSeconds: 5 diff --git a/kubernetes/loculus/values_preview_server.yaml b/kubernetes/loculus/values_preview_server.yaml index a7acaa3523..c6a1ea97d2 100644 --- a/kubernetes/loculus/values_preview_server.yaml +++ b/kubernetes/loculus/values_preview_server.yaml @@ -140,4 +140,4 @@ defaultResources: cpu: "20m" limits: memory: "1Gi" - cpu: "20m" \ No newline at end of file + cpu: "20m" diff --git a/preprocessing/dummy/lineage-alternative.yaml b/preprocessing/dummy/lineage-alternative.yaml index b2b34e1fa8..8f7008394d 100644 --- a/preprocessing/dummy/lineage-alternative.yaml +++ b/preprocessing/dummy/lineage-alternative.yaml @@ -4,30 +4,30 @@ A: A.1: aliases: [] parents: - - A + - A A.2: aliases: [] parents: - - A + - A A.1.1: aliases: [] parents: - - A.1 - - A.2 + - A.1 + - A.2 B: aliases: [] parents: [] B.1: aliases: [] parents: - - B + - B B.1.1: aliases: - - C + - C parents: - - B.1 + - B.1 C.1: aliases: - - B.1.1.1 + - B.1.1.1 parents: - - C + - C diff --git a/preprocessing/dummy/lineage.yaml b/preprocessing/dummy/lineage.yaml index 0c533e1a6c..52236c2360 100644 --- a/preprocessing/dummy/lineage.yaml +++ b/preprocessing/dummy/lineage.yaml @@ -4,13 +4,13 @@ A: A.1: aliases: [] parents: - - A + - A A.1.1: aliases: - - B + - B parents: - - A.1 + - A.1 A.2: aliases: [] parents: - - A + - A diff --git a/preprocessing/nextclade/tests/embl_required_metadata.yaml b/preprocessing/nextclade/tests/embl_required_metadata.yaml index 919e8f94e9..18641f04a9 100644 --- a/preprocessing/nextclade/tests/embl_required_metadata.yaml +++ b/preprocessing/nextclade/tests/embl_required_metadata.yaml @@ -20,4 +20,4 @@ processing_spec: type: date function: parse_and_assert_past_date inputs: - date: sampleCollectionDate \ No newline at end of file + date: sampleCollectionDate diff --git a/preprocessing/nextclade/tests/multi_reference_config.yaml b/preprocessing/nextclade/tests/multi_reference_config.yaml index 5784f1e403..f2f2aae0b9 100644 --- a/preprocessing/nextclade/tests/multi_reference_config.yaml +++ b/preprocessing/nextclade/tests/multi_reference_config.yaml @@ -7,14 +7,14 @@ nextclade_dataset_server: TEST segments: - name: main references: - - name: ebola-sudan - nextclade_dataset_name: ebola-dataset/ebola-sudan - accepted_dataset_matches: [ebola-sudan] - genes: [NPEbolaSudan, VP35EbolaSudan] # Names of genes in nextclade dataset gff3 file - - name: ebola-zaire - nextclade_dataset_name: ebola-dataset/ebola-zaire - accepted_dataset_matches: [ebola-zaire] - genes: [VP24EbolaZaire, LEbolaZaire] + - name: ebola-sudan + nextclade_dataset_name: ebola-dataset/ebola-sudan + accepted_dataset_matches: [ebola-sudan] + genes: [NPEbolaSudan, VP35EbolaSudan] # Names of genes in nextclade dataset gff3 file + - name: ebola-zaire + nextclade_dataset_name: ebola-dataset/ebola-zaire + accepted_dataset_matches: [ebola-zaire] + genes: [VP24EbolaZaire, LEbolaZaire] organism: multi-ebola-test processing_spec: subtype: diff --git a/preprocessing/nextclade/tests/multi_segment_config.yaml b/preprocessing/nextclade/tests/multi_segment_config.yaml index 2a9fa7a5b9..cfded28c62 100644 --- a/preprocessing/nextclade/tests/multi_segment_config.yaml +++ b/preprocessing/nextclade/tests/multi_segment_config.yaml @@ -6,12 +6,12 @@ segment_classification_method: "minimizer" segments: - name: ebola-sudan references: - - nextclade_dataset_name: ebola-dataset/ebola-sudan - genes: [NPEbolaSudan, VP35EbolaSudan] + - nextclade_dataset_name: ebola-dataset/ebola-sudan + genes: [NPEbolaSudan, VP35EbolaSudan] - name: ebola-zaire references: - - nextclade_dataset_name: ebola-dataset/ebola-zaire - genes: [VP24EbolaZaire, LEbolaZaire] + - nextclade_dataset_name: ebola-dataset/ebola-zaire + genes: [VP24EbolaZaire, LEbolaZaire] organism: multi-ebola-test processing_spec: totalInsertedNucs_ebola-zaire: diff --git a/preprocessing/nextclade/tests/multi_segment_multi_reference.yaml b/preprocessing/nextclade/tests/multi_segment_multi_reference.yaml index fea2a6e783..b5f69f9702 100644 --- a/preprocessing/nextclade/tests/multi_segment_multi_reference.yaml +++ b/preprocessing/nextclade/tests/multi_segment_multi_reference.yaml @@ -8,20 +8,20 @@ diamond_dmnd_url: FAKE_URL segments: - name: L references: - - name: singleReference - nextclade_dataset_name: CCHF/L - accepted_dataset_matches: [L] - genes: [RdRp] + - name: singleReference + nextclade_dataset_name: CCHF/L + accepted_dataset_matches: [L] + genes: [RdRp] - name: S references: - - name: 1and6 - nextclade_dataset_name: CCHF/S-1and6 - accepted_dataset_matches: [S-1and6] - genes: [NP] - - name: 2to5 - nextclade_dataset_name: CCHF/S-2to5 - accepted_dataset_matches: [S-2to5] - genes: [NP] + - name: 1and6 + nextclade_dataset_name: CCHF/S-1and6 + accepted_dataset_matches: [S-1and6] + genes: [NP] + - name: 2to5 + nextclade_dataset_name: CCHF/S-2to5 + accepted_dataset_matches: [S-2to5] + genes: [NP] organism: multi-reference-cchf-test processing_spec: subtype_S: diff --git a/preprocessing/nextclade/tests/no_alignment_config.yaml b/preprocessing/nextclade/tests/no_alignment_config.yaml index 569a649a5f..da2eb5c58c 100644 --- a/preprocessing/nextclade/tests/no_alignment_config.yaml +++ b/preprocessing/nextclade/tests/no_alignment_config.yaml @@ -87,7 +87,7 @@ processing_spec: extracted_regex_field: function: extract_regex args: - pattern: "^EPI_ISL_(?P[0-9]+)?$" # Note the optional group to allow for testing of missing values + pattern: "^EPI_ISL_(?P[0-9]+)?$" # Note the optional group to allow for testing of missing values capture_group: id inputs: regex_field: regex_field diff --git a/preprocessing/nextclade/tests/single_segment_config.yaml b/preprocessing/nextclade/tests/single_segment_config.yaml index 408f0beb41..3c8f31bcc4 100644 --- a/preprocessing/nextclade/tests/single_segment_config.yaml +++ b/preprocessing/nextclade/tests/single_segment_config.yaml @@ -8,8 +8,8 @@ db_name: "Loculus" segments: - name: main references: - - nextclade_dataset_name: ebola-sudan - genes: [NPEbolaSudan, VP35EbolaSudan] + - nextclade_dataset_name: ebola-sudan + genes: [NPEbolaSudan, VP35EbolaSudan] organism: ebola-sudan-test processing_spec: completeness: From 6471204783282f5b387b4b2555e6dd6f80a9ac94 Mon Sep 17 00:00:00 2001 From: tombch Date: Fri, 6 Mar 2026 12:14:59 +0100 Subject: [PATCH 10/10] Pinned yamlfmt to latest version 0.21.0 --- .github/workflows/yamlfmt-lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/yamlfmt-lint.yml b/.github/workflows/yamlfmt-lint.yml index 23aed02536..9bde9939ff 100644 --- a/.github/workflows/yamlfmt-lint.yml +++ b/.github/workflows/yamlfmt-lint.yml @@ -10,7 +10,7 @@ jobs: lint: runs-on: ubuntu-latest container: - image: ghcr.io/google/yamlfmt:latest + image: ghcr.io/google/yamlfmt:0.21.0 steps: - name: Checkout repository uses: actions/checkout@v6