fix: optimize workflow resource usage

- Use rust-htslib for SAM parsing.
- Switch from uv -> poetry.
- Add or replace many Rust unit and integration tests.
- Improve container-based testing setup.
- Add testing for final results report sent to Virtool.
- Don't write subtracted_read_ids.txt to the working directory.
- Improve separation of PyO3 and business logic.
- Fix bug in  best hit inconsistency with pathoscope2.
- Remove unused sam_50.sam.
- Simplify or improve performance of most modules.
- Don't rewrite a SAM file after EM. Calculate coverage and other output directly from
memory.
- Prevent float precision flakiness by rounding to 10 digits.
- Reorganize Rust codebase.

Author: igboyes

.deepsource.toml

@@ -11,4 +11,10 @@ name = "python"
 enabled = true
 
   [analyzers.meta]
-  runtime_version = "3.x.x"
+  runtime_version = "3.x.x"
+
+[[analyzers]]
+name = "rust"
+
+  [analyzers.meta]
+  msrv = "stable"

.dockerignore

@@ -1,3 +1,38 @@
-.github
-subtracted_read_ids.txt
+# Git
+.git
+.gitignore
+
+# Build artifacts (handled by named volumes)
+.venv
 target
+node_modules
+
+# IDE and editor files
+.vscode
+.idea
+*.swp
+*.swo
+*~
+
+# OS files
+.DS_Store
+Thumbs.db
+
+# Logs and temporary files
+*.log
+*.tmp
+.cache
+subtracted_read_ids.txt
+
+# Python cache
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+
+# Rust artifacts
+rust.abi3.so
+
+# Docker files (avoid recursion)
+Dockerfile
+.dockerignore

.github/workflows/ci.yml

@@ -39,30 +39,37 @@ jobs:
           tags: virtool/pathoscope:test
           cache-from: type=gha
           cache-to: type=gha,mode=max
-  test:
-    name: Test
+  test-rust:
+    name: Test / Rust
     runs-on: ubuntu-24.04
+    permissions:
+      contents: read
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Setup Docker
         uses: docker/setup-buildx-action@v3
-      - name: Build
-        id: build
-        uses: docker/build-push-action@v5
-        with:
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          context: .
-          load: true
-          target: test
-      - name: Test
-        run: docker run --rm -t ${{ steps.build.outputs.imageid }} poetry run pytest
+      - name: Setup mise
+        uses: jdx/mise-action@v2
+      - name: Test Rust
+        run: mise run test:rust
+  test-python:
+    name: Test / Python
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Docker
+        uses: docker/setup-buildx-action@v3
+      - name: Setup mise
+        uses: jdx/mise-action@v2
+      - name: Test Python
+        run: mise run test:python
   release:
     name: "Publish / Release"
     runs-on: ubuntu-24.04
     if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
-    needs: [build, commitlint, test]
+    needs: [build, commitlint, test-rust, test-python]
     outputs:
       git-tag: ${{ steps.semantic.outputs.git-tag }}
     steps:

.gitignore

@@ -4,6 +4,7 @@
 .vscode
 *.pyc
 __pycache__
-subtracted_read_ids.txt
-target
 CLAUDE.md
+Pathoscope
+rust.abi3.so
+target

.mise.toml

@@ -1,26 +1,58 @@
 [tools]
 python = "3.12"
-poetry = "latest"
+uv = "latest"
 
-[tasks.test-setup]
-description = "Build base Docker image for testing"
-run = "docker build --target dev -t pathoscope:dev ."
+[vars]
+dev_exec = "docker compose exec dev"
+
+[tasks."dev:start"]
+description = "Start the dev container"
+run = "docker compose up -d dev"
+
+[tasks."dev:stop"]
+description = "Stop the dev container"
+run = "docker compose stop dev"
+
+[tasks."test:rust"]
+description = "Run Rust unit tests in container"
+depends = ["dev:start"]
+run = [
+  "{{vars.dev_exec}} bash -c 'uv sync && uv run maturin develop --release'",
+  '{{vars.dev_exec}} cargo test --lib {{arg(name="cargo_args", var=true, default="")}}'
+]
+
+[tasks."test:python"]
+description = "Run Python/e2e tests in container"
+depends = ["dev:start"]
+run = [
+  "{{vars.dev_exec}} bash -c 'uv sync && uv run maturin develop --release'",
+  '{{vars.dev_exec}} uv run pytest {{arg(name="pytest_args", var=true, default="")}}'
+]
+
+[tasks.clippy]
+description = "Run clippy linter in container"
+run = "{{vars.dev_exec}} cargo clippy -- -W clippy::all"
 
 [tasks.test]
-description = "Run tests in Docker"
-depends = ["test-setup"]
-run = "./scripts/test-docker.sh"
-
-[tasks.test-watch]
-description = "Run tests in watch mode"
-depends = ["test-setup"]
-run = "./scripts/test-docker.sh --watch"
-
-[tasks.test-shell]
-description = "Interactive shell in test container"
-depends = ["test-setup"]
-run = "./scripts/test-docker.sh --shell"
-
-[tasks.test-clean]
-description = "Clean up test Docker resources"
-run = "docker rmi pathoscope:dev 2>/dev/null || true"
+description = "Run all tests (Rust + Python)"
+depends = ["dev:start"]
+run = [
+  "mise run test:rust",
+  "mise run test:python"
+]
+
+[tasks.dev]
+description = "Interactive shell in dev container"
+run = "{{vars.dev_exec}} bash"
+
+[tasks."dev:clean"]
+description = "Clean up all dev containers and volumes"
+run = "docker compose down -v --remove-orphans"
+
+[tasks."dev:rebuild"]
+description = "Force rebuild the dev container"
+run = [
+  "docker compose down dev",
+  "docker compose build --no-cache dev",
+  "docker compose up -d dev"
+]