fix(deps): use python 3.12

* Update Rust complication strategy.
* Reorganize code and tests.
* Use latest Maturin and update Rust compilation strategy.
* Use latest Bowtie2.

Author: igboyes

.devcontainer/Dockerfile

@@ -0,0 +1,26 @@
+FROM debian:bookworm as bowtie2
+WORKDIR /build
+RUN apt-get update && apt-get install -y build-essential cmake wget zlib1g-dev
+RUN wget https://github.com/BenLangmead/bowtie2/archive/refs/tags/v2.5.4.tar.gz
+RUN tar -xvf v2.5.4.tar.gz
+WORKDIR bowtie2-2.5.4
+RUN make
+RUN mkdir /build/bowtie2
+RUN cp bowtie2* /build/bowtie2/
+
+FROM debian:bookworm as pigz
+WORKDIR /build
+RUN apt-get update && apt-get install -y gcc make wget zlib1g-dev
+RUN wget https://zlib.net/pigz/pigz-2.8.tar.gz && \
+    tar -xzvf pigz-2.8.tar.gz && \
+    cd pigz-2.8 && \
+    make
+
+FROM python:3.12-bookworm
+COPY --from=bowtie2 /build/bowtie2/* /usr/local/bin/
+COPY --from=ghcr.io/virtool/workflow-tools:2.0.1 /opt/fastqc /opt/fastqc
+COPY --from=ghcr.io/virtool/workflow-tools:2.0.1 /opt/hmmer /opt/hmmer
+COPY --from=ghcr.io/virtool/workflow-tools:2.0.1 /usr/local/bin/pigz /usr/local/bin/
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+RUN curl -sSL https://install.python-poetry.org | python -
+ENV PATH="/root/.local/bin:/root/.cargo/bin:${PATH}"

.devcontainer/devcontainer.json

@@ -0,0 +1,7 @@
+{
+  "name": "pathoscope-test",
+  "build": {
+    "dockerfile": "Dockerfile",
+    "context": "."
+  },
+}

.github/workflows/ci.yml

@@ -35,7 +35,7 @@ jobs:
           load: true
           target: test
       - name: Test
-        run: docker run --rm -t ${{ steps.build.outputs.imageid }} pytest
+        run: docker run --rm -t ${{ steps.build.outputs.imageid }} poetry run pytest
   release:
     runs-on: ubuntu-22.04
     needs: [commitlint, test]

.github/workflows/publish.yml

@@ -16,11 +16,9 @@ jobs:
     if: github.repository_owner == 'Virtool'
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Write VERSION file
         run: echo ${{ github.event.release.tag_name }} > VERSION
-      - name: Update pyproject.toml version
-        run: sed -i 's/0\.0\.0/${{ github.event.release.tag_name }}/' pyproject.toml
       - name: Login to Registry
         uses: docker/login-action@v3
         with:
@@ -33,11 +31,10 @@ jobs:
         with:
           images: ${{ env.REGISTRY }}/virtool/pathoscope
       - name: Build and Push
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
           labels: ${{ steps.meta.outputs.labels }}
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           target: "base"
-

Cargo.lock

@@ -1,17 +1,19 @@
 [package]
-    name = "rust_utils"
-    version = "0.1.0"
-    edition = "2021"
-    authors = ["Markus Swoveland"]
+name = "workflowpathoscope"
+version = "0.0.0"
+edition = "2021"
+authors = ["Markus Swoveland"]
+
+[lib]
+name = "workflow_pathoscope"
+# "cdylib" is necessary to produce a shared library for Python to import from.
+crate-type = ["cdylib"]
 
 [dependencies]
 yaml-rust = "0.4.5"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies.pyo3]
-version = "0.16.3"
-
-[features]
-extension-module = ["pyo3/extension-module"]
-default = ["extension-module"]
+version = "^0.20.0"
+# "abi3-py38" tells pyo3 (and maturin) to build using the stable ABI with minimum
+# Python version 3.12
+features = ["abi3-py312"]

Cargo.toml

@@ -1,46 +1,66 @@
-FROM debian:buster as prep
+FROM debian:bookworm as bowtie2
 WORKDIR /build
-RUN apt-get update && apt-get install -y make gcc zlib1g-dev wget unzip
+RUN apt-get update && apt-get install -y build-essential cmake wget zlib1g-dev
+RUN wget https://github.com/BenLangmead/bowtie2/archive/refs/tags/v2.5.4.tar.gz
+RUN tar -xvf v2.5.4.tar.gz
+WORKDIR bowtie2-2.5.4
+RUN make
+RUN mkdir /build/bowtie2
+RUN cp bowtie2* /build/bowtie2/
+
+FROM debian:bookworm as pigz
+WORKDIR /build
+RUN apt-get update && apt-get install -y gcc make wget zlib1g-dev
 RUN wget https://zlib.net/pigz/pigz-2.8.tar.gz && \
     tar -xzvf pigz-2.8.tar.gz && \
     cd pigz-2.8 && \
     make
-RUN wget https://www.bioinformatics.babraham.ac.uk/projects/fastqc/fastqc_v0.11.9.zip && \
-    unzip fastqc_v0.11.9.zip
-RUN wget https://github.com/BenLangmead/bowtie2/releases/download/v2.3.2/bowtie2-2.3.2-legacy-linux-x86_64.zip && \
-    unzip bowtie2-2.3.2-legacy-linux-x86_64.zip && \
-    mkdir bowtie2 && \
-    cp bowtie2-2.3.2-legacy/bowtie2* bowtie2
-
 
-FROM python:3.10-buster as base
+FROM python:3.12-bookworm as deps
 WORKDIR /app
-COPY --from=prep /build/bowtie2/* /usr/local/bin/
-COPY --from=prep /build/FastQC /opt/fastqc
-COPY --from=prep /build/pigz-2.8/pigz /usr/local/bin/pigz
-RUN chmod ugo+x /opt/fastqc/fastqc && \
-    ln -fs /opt/fastqc/fastqc /usr/local/bin/fastqc && \
-    for file in `ls /opt/hmmer/bin`; do ln -fs /opt/hmmer/bin/${file} /usr/local/bin/${file};  done
+COPY --from=bowtie2 /build/bowtie2/* /usr/local/bin/
+COPY --from=ghcr.io/virtool/workflow-tools:2.0.1 /opt/fastqc /opt/fastqc
+COPY --from=ghcr.io/virtool/workflow-tools:2.0.1 /opt/hmmer /opt/hmmer
+COPY --from=ghcr.io/virtool/workflow-tools:2.0.1 /usr/local/bin/pigz /usr/local/bin/
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends curl build-essential default-jre && \
+    apt-get install -y --no-install-recommends default-jre && \
     rm -rf /var/lib/apt/lists/* && \
     apt-get clean
+
+FROM python:3.12-bookworm as poetry
+WORKDIR /app
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
 RUN curl -sSL https://install.python-poetry.org | python -
-ENV PATH="/root/.cargo/bin:/root/.local/bin:${PATH}"
-RUN pip install --upgrade pip
-RUN pip install maturin==0.14.5
-COPY src src
-COPY Cargo.toml Cargo.lock fixtures.py utils.py  poetry.lock pyproject.toml workflow.py ./
-RUN maturin build --release
-RUN poetry export > requirements.txt
-RUN pip install -r requirements.txt
-RUN pip install /app/target/wheels/rust_utils*.whl
-COPY VERSION* ./
+ENV PATH="/root/.local/bin:/root/.cargo/bin:${PATH}" \
+    POETRY_CACHE_DIR='/tmp/poetry_cache' \
+    POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1
+COPY Cargo.toml Cargo.lock poetry.lock pyproject.toml ./
+COPY python/ ./python
+COPY src ./src
+RUN poetry install
+RUN poetry run maturin develop --release
 
-FROM base as test
+FROM deps as base
 WORKDIR /app
-RUN poetry export --with dev > requirements.txt
-RUN pip install -r requirements.txt
+ENV VIRTUAL_ENV=/app/.venv \
+    PATH="/app/.venv/bin:/opt/fastqc:/opt/hmmer/bin:${PATH}"
+RUN chmod ugo+x /opt/fastqc/fastqc
+COPY --from=poetry /app/.venv /app/.venv
+COPY --from=poetry /app/python /app/python
+COPY fixtures.py workflow.py VERSION* ./
+
+FROM deps as test
+WORKDIR /app
+ENV PATH="/root/.local/bin:/root/.cargo/bin:${PATH}"
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+RUN curl -sSL https://install.python-poetry.org | python -
+COPY Cargo.lock Cargo.toml pyproject.toml poetry.lock ./
+COPY src ./src
+COPY python ./python
+RUN poetry install
+RUN poetry run maturin develop --release
 COPY example ./example
 COPY tests ./tests
+COPY fixtures.py workflow.py VERSION* ./