ci: Use nox as the task runner (#4432)

* ci: Use nox as the task runner Signed-off-by: Frost Ming <[email protected]> * fix run command Signed-off-by: Frost Ming <[email protected]> * fix test Signed-off-by: Frost Ming <[email protected]> * install by pipx Signed-off-by: Frost Ming <[email protected]> * don't install requirements in tests Signed-off-by: Frost Ming <[email protected]> * change to session fixture Signed-off-by: Frost Ming <[email protected]> * fix: update transitive dependencies Signed-off-by: Frost Ming <[email protected]> * pin tensorflow version Signed-off-by: Frost Ming <[email protected]> * fix: reset config after test Signed-off-by: Frost Ming <[email protected]> * fix test requirements Signed-off-by: Frost Ming <[email protected]> * fix the stage name Signed-off-by: Frost Ming <[email protected]>
bentoml · Feb 1, 2024 · a432216 · a432216
1 parent 2cc2c54
commit a432216
Show file tree

Hide file tree

Showing 18 changed files with 771 additions and 6,470 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -35,18 +35,15 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0 # fetch all tags and branches
-      - name: Set up PDM
-        uses: pdm-project/setup-pdm@v3
+      - name: Set up Python
+        uses: actions/setup-python@v5
         with:
-          cache: true
+          cache: pip
           python-version: ${{ matrix.python-version }}
-          enable-pep582: false
       - name: Install dependencies
-        run: |
-          pdm venv create --with-pip --force
-          pdm sync -d -G grpc -G io -G testing
+        run: pipx install pdm && pipx install nox
       - name: Unit tests
-        run: pdm run -v unit
+        run: nox --session unit-${{ matrix.python-version }}
       - name: Disambiguate coverage filename
         run: mv .coverage ".coverage.unit.${{ matrix.os }}.${{ matrix.python-version }}"
       - name: Upload coverage data
@@ -86,19 +83,15 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0 # fetch all tags and branches
-      - name: Set up PDM
-        uses: pdm-project/setup-pdm@v3
+      - name: Set up Python
+        uses: actions/setup-python@v5
         with:
           python-version: '3.8'
-          enable-pep582: false
+          cache: pip
       - name: Install dependencies
-        run: |
-          pdm venv create --with-pip
-          pdm sync -d -G testing -G frameworks
-          pdm run pip install torch torchaudio torchvision --index-url https://download.pytorch.org/whl/cpu
-          pdm run pip install -U tensorflow onnx onnxruntime "jax[cpu]"
+        run: pipx install pdm && pipx install nox
       - name: Run framework integration tests
-        run: pdm run -v integration "${{ matrix.framework }}"
+        run: nox --session "framework-integration(framework='${{ matrix.framework }}')"
       - name: Disambiguate coverage filename
         run: mv .coverage ".coverage.integrations.ubuntu-latest.3.8.${{ matrix.framework }}"
       - name: Upload coverage data
@@ -128,32 +121,30 @@ jobs:
         if: ${{ matrix.os == 'ubuntu-latest' }}
         id: buildx
         uses: docker/setup-buildx-action@v3
-      - name: Set up PDM
-        uses: pdm-project/setup-pdm@v3
+      - name: Set up Python
+        uses: actions/setup-python@v5
         with:
-          cache: true
+          cache: pip
           python-version: ${{ matrix.python-version }}
-          enable-pep582: false
       - name: Install dependencies
-        run: pdm sync -d -G io -G testing -G monitor-otlp
+        run: pipx install pdm && pipx install nox
       - name: Run the monitoring tests
-        working-directory: examples/monitoring/task_classification
-        run: mkdir -p monitoring && pdm run -v monitoring
+        run: nox -s e2e-monitoring-${{ matrix.python-version }}
       - name: Disambiguate coverage filename
         run: mv .coverage ".coverage.monitoring.${{ matrix.os }}.${{ matrix.python-version }}"
       - name: Upload coverage data
         uses: actions/upload-artifact@v3
         with:
           name: coverage-monitoring-data
           path: .coverage.*
-  e2e-bento-server:
+  e2e-test:
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
         python-version: ['3.8', '3.11']
-        server_type: ['http']
-    name: ${{ matrix.server_type }}-e2e-tests (python${{ matrix.python-version }}.${{ matrix.os }})
+        suite: ['bento_server_http', 'bento_new_sdk']
+    name: ${{ matrix.suite }}-e2e-tests (python${{ matrix.python-version }}.${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     timeout-minutes: 90
     env:
@@ -169,61 +160,27 @@ jobs:
         if: ${{ matrix.os == 'ubuntu-latest' }}
         id: buildx
         uses: docker/setup-buildx-action@v3
-      - name: Set up PDM
-        uses: pdm-project/setup-pdm@v3
+      - name: Set up Python
+        uses: actions/setup-python@v5
         with:
-          cache: true
+          cache: pip
           python-version: ${{ matrix.python-version }}
-          enable-pep582: false
       - name: Install dependencies
-        run: |
-          pdm venv create --with-pip --force
-          pdm run pip install -r tests/e2e/bento_server_${{ matrix.server_type }}/requirements.txt
-          pdm sync -d -G io -G testing
-      - name: Run ${{ matrix.server_type }} tests
-        run: pdm run -v tests tests/e2e/bento_server_${{ matrix.server_type }}
+        run: pipx install pdm && pipx install nox
+      - name: Run ${{ matrix.suite }} tests
+        run: nox -s "e2e-testing-${{ matrix.python-version }}(suite='${{ matrix.suite }}')"
       - name: Disambiguate coverage filename
-        run: mv .coverage ".coverage.e2e.${{ matrix.os }}.${{ matrix.python-version }}"
+        run: mv .coverage ".coverage.e2e.${{ matrix.os }}.${{ matrix.python-version }}.${{ matrix.suite }}"
       - name: Upload coverage data
         uses: actions/upload-artifact@v3
         with:
           name: coverage-e2e-data
           path: .coverage.*
-  e2e-new-sdk:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-        python-version: ['3.8', '3.11']
-    name: new-sdk-e2e-tests (python${{ matrix.python-version }}.${{ matrix.os }})
-    runs-on: ${{ matrix.os }}
-    timeout-minutes: 90
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0 # fetch all tags and branches
-      - name: Set up PDM
-        uses: pdm-project/setup-pdm@v3
-        with:
-          cache: true
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: |
-          pdm venv create --with-pip --force
-          pdm sync -L pdm.ionext.lock
-          pdm run pip install -r tests/e2e/bento_new_sdk/requirements.txt
-      - name: Run tests
-        run: pdm run -v tests tests/e2e/bento_new_sdk
-      - name: Upload coverage data
-        uses: actions/upload-artifact@v3
-        with:
-          name: coverage-e2e-data
-          path: .coverage
   coverage:
     name: report-coverage
     runs-on: ubuntu-latest
     needs:
-      - e2e-bento-server
+      - e2e-test
       - e2e-monitoring
       - unit
       - integrations
@@ -232,12 +189,11 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: Set up PDM
-        uses: pdm-project/setup-pdm@v3
+      - name: Set up Python
+        uses: actions/setup-python@v5
         with:
-          cache: true
+          cache: pip
           python-version: '3.11'
-          enable-pep582: false
       - name: Download e2e coverage
         uses: actions/download-artifact@v3
         with:
@@ -255,19 +211,14 @@ jobs:
         with:
           name: coverage-unit-data
       - name: Install dependencies
-        run: pdm sync -d -G testing
-      - name: Combine coverage data
-        run: pdm run -v coverage-combine
+        run: pipx install pdm && pipx install nox
       - name: Export coverage reports and generate summary
-        run: pdm run -v coverage-generate-reports
+        run: nox -s coverage
       - name: Upload uncovered HTML report
         uses: actions/upload-artifact@v3
         with:
           name: uncovered-html-report
           path: htmlcov
-      - name: Write coverage summary report
-        if: github.event_name == 'pull_request'
-        run: pdm run -v coverage-write-report
       - name: Update coverage pull request comment
         if: github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork
         uses: marocchino/sticky-pull-request-comment@v2
@@ -277,7 +228,7 @@ jobs:
     if: github.event_name == 'pull_request'
     needs:
       - coverage
-      - e2e-bento-server
+      - e2e-test
       - e2e-monitoring
       - unit
       - integrations

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -12,10 +12,8 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: '3.10'
-      - name: Install dependencies
-        run: python -m pip install --upgrade build
       - name: Build
-        run: python -m build
+        run: pipx run build
       - name: Archive
         uses: actions/upload-artifact@v3
         with:

diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md
@@ -279,10 +279,10 @@ docker:
 
 ## Testing
 
-Make sure to install all test dependencies:
+Make sure to install all dev dependencies:
 
 ```bash
-pdm install -G testing -G grpc -G io
+pdm install
 ```
 
 BentoML tests come with a Pytest plugin. Export `PYTEST_PLUGINS`:
@@ -294,7 +294,7 @@ export PYTEST_PLUGINS=bentoml.testing.pytest.plugin
 To run all tests with PDM, do the following:
 
 ```bash
-pdm run all
+pdm run nox
 ```
 
 ### Adding new test suite

diff --git a/Makefile b/Makefile
@@ -41,8 +41,6 @@ clean: ## Clean all generated files
 	@find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
 
 # Docs
-watch-docs: ## Build and watch documentation
-	pdm run sphinx-autobuild docs/source docs/build/html --watch $(GIT_ROOT)/src/ --ignore "bazel-*"
 spellcheck-docs: ## Spell check documentation
 	pdm run sphinx-build -b spelling ./docs/source ./docs/build || (echo "Error running spellchecker.. You may need to run 'make install-spellchecker-deps'"; exit 1)
 OS := $(shell uname)

diff --git a/examples/monitoring/task_classification/tests/conftest.py b/examples/monitoring/task_classification/tests/conftest.py
@@ -14,26 +14,14 @@
 from bentoml._internal.configuration.containers import BentoMLContainer
 
 if t.TYPE_CHECKING:
-    from _pytest.config import Config
     from _pytest.fixtures import FixtureRequest
-    from _pytest.main import Session
-    from _pytest.nodes import Item
     from _pytest.tmpdir import TempPathFactory
 
 PROJECT_DIR = Path(__file__).parent.parent
 
 
-def pytest_collection_modifyitems(session: Session, config: Config, items: list[Item]):
-    subprocess.check_call(
-        [
-            sys.executable,
-            "-m",
-            "pip",
-            "install",
-            "-r",
-            f"{os.path.join(PROJECT_DIR, 'requirements.txt')}",
-        ]
-    )
+@pytest.fixture(scope="session", autouse=True)
+def prepare_model() -> None:
     try:
         print(f"Found {bentoml.models.get('iris_clf')}, skipping model saving.")
     except bentoml.exceptions.NotFound:
@@ -51,7 +39,7 @@ def fixture_monitoring_type(request: FixtureRequest) -> str:
 
 
 @pytest.fixture(name="monitoring_dir", scope="session")
-def fixture_monitoring_dir(tmp_path_factory: TempPathFactory) -> str:
+def fixture_monitoring_dir(tmp_path_factory: TempPathFactory) -> Path:
     d = tmp_path_factory.mktemp("monitoring")
     os.environ["MONITORING_LOG_PATH"] = d.__fspath__()
     return d