diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ce009b7272..defccfee1f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,17 +2,15 @@ name: CI on: push: - branches: - - main + branches: [main] pull_request: schedule: - cron: "0 0 * * 1/2" env: - BENTOML_DO_NOT_TRACK: True LINES: 120 COLUMNS: 120 - PYTEST_PLUGINS: bentoml.testing.pytest.plugin + BENTOML_DO_NOT_TRACK: True # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#defaultsrun defaults: @@ -26,6 +24,7 @@ jobs: related: ${{ steps.filter.outputs.related }} bentoml: ${{ steps.filter.outputs.bentoml }} docs: ${{ steps.filter.outputs.docs }} + protos: ${{ steps.filter.outputs.protos }} steps: - uses: actions/checkout@v3 - uses: dorny/paths-filter@v2 @@ -37,8 +36,6 @@ jobs: - .github/workflows/ci.yml - codecov.yml - pyproject.toml - - scripts/ci/config.yml - - scripts/ci/run_tests.sh - requirements/tests-requirements.txt protos: &protos - "src/bentoml/grpc/**/*.proto" @@ -100,14 +97,22 @@ jobs: pip install -r requirements/dev-requirements.txt - name: Format check - run: make ci-format + run: | + black --check src examples tests + black --check --pyi typings + isort --check . - name: Lint check - run: make ci-lint + run: | + git fetch origin "$GITHUB_BASE_REF" + git diff --name-only --diff-filter=d "origin/$GITHUB_BASE_REF" -z -- '*.py' | xargs -0 --no-run-if-empty pylint --rcfile pyproject.toml --fail-under 9.0 --exit-zero - name: Type check - run: make ci-pyright + run: | + git fetch origin "$GITHUB_BASE_REF" + git diff --name-only --diff-filter=d "origin/$GITHUB_BASE_REF" -z -- '*.py[i]' | xargs -0 --no-run-if-empty pyright - name: Proto check if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.protos == 'true') || github.event_name == 'push' }} - run: buf lint --config "src/bentoml/grpc/buf.yaml" --error-format msvs src + run: | + buf lint --config "src/bentoml/grpc/buf.yaml" --error-format msvs src documentation_spelling_check: runs-on: ubuntu-latest @@ -193,23 +198,21 @@ jobs: - name: Run unit tests run: | - OPTS=(--cov-config pyproject.toml --cov-report=xml:unit.xml -vvv) + OPTS=(--cov-config pyproject.toml --cov=src/bentoml --cov-append) if [ "${{ matrix.os }}" != 'windows-latest' ]; then # we will use pytest-xdist to improve tests run-time. OPTS=(${OPTS[@]} --dist loadfile -n auto --run-grpc-tests) fi # Now run the unit tests - python -m pytest tests/unit "${OPTS[@]}" + coverage run -m pytest tests/unit "${OPTS[@]}" + + - name: Generage coverage + run: coverage xml - name: Upload test coverage to Codecov uses: codecov/codecov-action@v3 with: - name: codecov-${{ matrix.os }}-python${{ matrix.python-version }} - fail_ci_if_error: true - flags: unit-tests - directory: ./ - files: ./unit.xml - verbose: true + files: coverage.xml token: ${{ secrets.CODECOV_TOKEN }} bento_server_e2e_tests: @@ -278,17 +281,17 @@ jobs: fi - name: Run ${{ matrix.server_type }} tests and generate coverage report - run: ./scripts/ci/run_tests.sh ${{ matrix.server_type }}_server --verbose + run: | + OPTS=(--cov-config pyproject.toml --cov=src/bentoml --cov-append) + coverage run -m pytest tests/e2e/bento_server_${{ matrix.server_type }} "${OPTS[@]}" + + - name: Generage coverage + run: coverage xml - name: Upload test coverage to Codecov uses: codecov/codecov-action@v3 with: - flags: e2e-tests-${{ matrix.server_type }} - name: codecov-${{ matrix.os }}-python${{ matrix.python-version }}-e2e - fail_ci_if_error: true - directory: ./ - files: ./tests/e2e/bento_server_${{ matrix.server_type }}/${{ matrix.server_type }}_server.xml - verbose: true + files: coverage.xml token: ${{ secrets.CODECOV_TOKEN }} concurrency: diff --git a/.github/workflows/frameworks.yml b/.github/workflows/frameworks.yml index c8e79c7830b..fd1b020fc44 100644 --- a/.github/workflows/frameworks.yml +++ b/.github/workflows/frameworks.yml @@ -2,49 +2,37 @@ name: Frameworks on: push: - branches: - - main + branches: [main] pull_request: - branches: - - main + branches: [main] schedule: - cron: "0 0 * * 1/2" env: - BENTOML_DO_NOT_TRACK: True LINES: 120 COLUMNS: 120 - PYTEST_PLUGINS: bentoml.testing.pytest.plugin + BENTOML_DO_NOT_TRACK: True + +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#defaultsrun +defaults: + run: + shell: bash --noprofile --norc -exo pipefail {0} jobs: diff: runs-on: ubuntu-latest outputs: - dependencies: ${{ steps.filter.outputs.dependencies }} catboost: ${{ steps.filter.outputs.catboost }} - detectron2: ${{ steps.filter.outputs.detectron2 }} - easyocr: ${{ steps.filter.outputs.easyocr }} - evalml: ${{ steps.filter.outputs.evalml }} fastai: ${{ steps.filter.outputs.fastai }} - fasttext: ${{ steps.filter.outputs.fasttext }} - gluon: ${{ steps.filter.outputs.gluon }} - h2o: ${{ steps.filter.outputs.h2o }} - keras_tf2: ${{ steps.filter.outputs.keras_tf2 }} + keras: ${{ steps.filter.outputs.keras }} lightgbm: ${{ steps.filter.outputs.lightgbm }} mlflow: ${{ steps.filter.outputs.mlflow }} onnx: ${{ steps.filter.outputs.onnx }} - onnxmlir: ${{ steps.filter.outputs.onnxmlir }} - paddle: ${{ steps.filter.outputs.paddle }} picklable_model: ${{ steps.filter.outputs.picklable_model }} - pycaret: ${{ steps.filter.outputs.pycaret }} - pyspark_mllib: ${{ steps.filter.outputs.pyspark_mllib }} pytorch: ${{ steps.filter.outputs.pytorch }} pytorch_lightning: ${{ steps.filter.outputs.pytorch_lightning }} sklearn: ${{ steps.filter.outputs.sklearn }} - spacy: ${{ steps.filter.outputs.spacy }} - statsmodels: ${{ steps.filter.outputs.statsmodels}} - tf1: ${{ steps.filter.outputs.tf1 }} - tf2: ${{ steps.filter.outputs.tf2 }} + tensorflow: ${{ steps.filter.outputs.tensorflow }} torchscript: ${{ steps.filter.outputs.torchscript }} transformers: ${{ steps.filter.outputs.transformers }} xgboost: ${{ steps.filter.outputs.xgboost }} @@ -54,170 +42,91 @@ jobs: id: filter with: filters: | - dependencies: &dependencies - - pyproject.toml - - scripts/ci/config.yml - - requirements/tests-requirements.txt related: &related - - *dependencies - codecov.yml + - pyproject.toml + - requirements/tests-requirements.txt - .github/workflows/frameworks.yml - - scripts/ci/run_tests.sh + - tests/integration/frameworks/conftest.py - tests/integration/frameworks/test_frameworks.py - runner: &runner - - *related - src/bentoml/_internal/runner/** - src/bentoml/_internal/models/** catboost: - - *runner + - *related - src/bentoml/catboost.py - src/bentoml/_internal/frameworks/catboost.py - tests/integration/frameworks/models/catboost.py - - tests/integration/frameworks/test_frameworks.py - detectron2: - - *runner - - src/bentoml/detectron.py - - src/bentoml/_internal/frameworks/detectron.py - - tests/integration/frameworks/test_detectron2_impl.py - easyocr: - - *runner - - src/bentoml/easyocr.py - - src/bentoml/_internal/frameworks/easyocr.py - - tests/integration/frameworks/test_easyocr_impl.py - evalml: - - *runner - - src/bentoml/evalml.py - - src/bentoml/_internal/frameworks/evalml.py - - tests/integration/frameworks/test_evalml_impl.py - fasttext: - - *runner - - src/bentoml/fasttext.py - - src/bentoml/_internal/frameworks/fasttext.py - - tests/integration/frameworks/test_fasttext_impl.py - gluon: - - *runner - - src/bentoml/gluon.py - - src/bentoml/_internal/frameworks/gluon.py - - tests/integration/frameworks/test_gluon_impl.py - h2o: - - *runner - - src/bentoml/h2o.py - - src/bentoml/_internal/frameworks/h2o.py - - tests/integration/frameworks/test_h2o_impl.py lightgbm: - - *runner + - *related - src/bentoml/lightgbm.py - src/bentoml/_internal/frameworks/lightgbm.py - tests/integration/frameworks/models/lightgbm.py - - tests/integration/frameworks/test_frameworks.py mlflow: - - *runner + - *related - src/bentoml/mlflow.py - src/bentoml/_internal/frameworks/mlflow.py - tests/integration/frameworks/mlflow fastai: - - *runner + - *related - src/bentoml/fastai.py - src/bentoml/_internal/frameworks/fastai.py - src/bentoml/_internal/frameworks/common/pytorch.py - - tests/integration/frameworks/test_frameworks.py - tests/integration/frameworks/test_fastai_unit.py onnx: - - *runner + - *related - src/bentoml/onnx.py - src/bentoml/_internal/frameworks/onnx.py - tests/integration/frameworks/models/onnx.py - - tests/integration/frameworks/test_frameworks.py - onnxmlir: - - *runner - - src/bentoml/onnxmlir.py - - src/bentoml/_internal/frameworks/onnxmlir.py - - tests/integration/frameworks/test_onnxmlir_impl.py - paddle: - - *runner - - src/bentoml/paddle.py - - src/bentoml/_internal/frameworks/paddle.py - - tests/integration/frameworks/paddle picklable_model: - - *runner + - *related - src/bentoml/picklable_model.py - src/bentoml/_internal/frameworks/picklable_model.py - - tests/integration/frameworks/picklable_model - pycaret: - - *runner - - src/bentoml/pycaret.py - - src/bentoml/_internal/frameworks/pycaret.py - - tests/integration/frameworks/test_pycaret_impl.py - pyspark_mllib: - - *runner - - src/bentoml/pyspark.py - - src/bentoml/_internal/frameworks/pyspark.py - - tests/integration/frameworks/test_pyspark_impl.py + - tests/integration/frameworks/models/picklable_model.py pytorch: - - *runner + - *related - src/bentoml/pytorch.py - - src/bentoml/_internal/frameworks/common/pytorch.py - src/bentoml/_internal/frameworks/pytorch.py + - src/bentoml/_internal/frameworks/common/pytorch.py - tests/integration/frameworks/test_pytorch_unit.py - - tests/integration/frameworks/test_frameworks.py torchscript: - - *runner + - *related - src/bentoml/torchscript.py - src/bentoml/_internal/frameworks/common/pytorch.py - src/bentoml/_internal/frameworks/torchscript.py - - tests/integration/frameworks/test_torchscript_impl.py pytorch_lightning: - - *runner + - *related - src/bentoml/pytorch.py - src/bentoml/pytorch_lightning.py - src/bentoml/_internal/frameworks/common/pytorch.py - src/bentoml/_internal/frameworks/torchscript.py - src/bentoml/_internal/frameworks/pytorch_lightning.py - - tests/integration/frameworks/test_pytorch_lightning_impl.py sklearn: - - *runner + - *related - src/bentoml/sklearn.py - src/bentoml/_internal/frameworks/sklearn.py - - tests/integration/frameworks/test_sklearn_impl.py - spacy: - - *runner - - src/bentoml/spacy.py - - src/bentoml/_internal/frameworks/spacy.py - - tests/integration/frameworks/spacy - statsmodels: - - *runner - - src/bentoml/statsmodels.py - - src/bentoml/_internal/frameworks/statsmodels.py - - tests/integration/frameworks/test_statsmodels_impl.py - tf1: &tf1 - - *runner - - src/bentoml/tensorflow.py - - src/bentoml/_internal/frameworks/tensorflow_v1.py - - tests/integration/frameworks/test_tensorflow_v1_impl.py - tf2: &tf2 - - *runner + - tests/integration/frameworks/models/sklearn.py + tensorflow: &tensorflow + - *related - src/bentoml/tensorflow.py - src/bentoml/_internal/frameworks/tensorflow_v2.py - - tests/integration/frameworks/models/tensorflow_v2.py - - tests/integration/frameworks/tensorflow_v2_unit.py - keras_tf2: - - *runner - - *tf2 + - tests/integration/frameworks/models/tensorflow.py + - tests/integration/frameworks/test_tensorflow_unit.py + keras: + - *related + - *tensorflow - src/bentoml/keras.py - src/bentoml/_internal/frameworks/keras.py - tests/integration/frameworks/models/keras.py - - tests/integration/frameworks/test_frameworks.py transformers: - - *runner + - *related - src/bentoml/transformers.py - src/bentoml/_internal/frameworks/transformers.py - - tests/integration/frameworks/test_transformers_impl.py + - tests/integration/frameworks/test_transformers_unit.py xgboost: - - *runner + - *related - src/bentoml/xgboost.py - src/bentoml/_internal/frameworks/xgboost.py - tests/integration/frameworks/models/xgboost.py - - tests/integration/frameworks/test_frameworks.py catboost_integration_tests: needs: diff @@ -247,117 +156,23 @@ jobs: - name: Install dependencies run: | pip install . + pip install catboost pip install -r requirements/tests-requirements.txt - name: Run tests and generate coverage report - run: make tests-catboost - shell: bash - - - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.catboost == 'true' }} - uses: codecov/codecov-action@v3 - with: - directory: ./ - files: ./catboost.xml - flags: catboost - verbose: true - token: ${{ secrets.CODECOV_TOKEN }} - - detectron2_integration_tests: - needs: diff - if: ${{ false }} - #if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.detectron2 == 'true') || github.event_name == 'push' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # fetch all tags and branches - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Get pip cache dir - id: cache-dir - run: | - echo ::set-output name=dir::$(pip cache dir) - - - name: Cache pip dependencies - uses: actions/cache@v3 - id: cache-pip - with: - path: ${{ steps.cache-dir.outputs.dir }} - key: ${{ runner.os }}-tests-${{ hashFiles('requirements/tests-requirements.txt') }} - - - name: Install dependencies - run: | - pip install . - pip install -r requirements/tests-requirements.txt - - - name: Run tests and generate coverage report - run: make tests-detectron2 - shell: bash - - - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.detectron2 == 'true' }} - uses: codecov/codecov-action@v3 - with: - directory: ./ - files: ./detectron2.xml - flags: detectron - verbose: true - token: ${{ secrets.CODECOV_TOKEN }} - - easyocr_integration_tests: - needs: diff - #if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.easyocr == 'true') || github.event_name == 'push' }} - if: ${{ false }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # fetch all tags and branches - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Get pip cache dir - id: cache-dir - run: | - echo ::set-output name=dir::$(pip cache dir) - - - name: Cache pip dependencies - uses: actions/cache@v3 - id: cache-pip - with: - path: ${{ steps.cache-dir.outputs.dir }} - key: ${{ runner.os }}-tests-${{ hashFiles('requirements/tests-requirements.txt') }} - - - name: Install dependencies run: | - pip install . - pip install -r requirements/tests-requirements.txt + OPTS=(--cov-config pyproject.toml --cov src/bentoml --cov-append --framework catboost) + coverage run -m pytest tests/integration/frameworks/test_frameworks.py "${OPTS[@]}" - - name: Run tests and generate coverage report - run: make tests-easyocr - shell: bash + - name: Generate coverage + run: coverage xml - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.easyocr == 'true' }} uses: codecov/codecov-action@v3 with: - directory: ./ - files: ./easyocr.xml - flags: easyocr - verbose: true + files: ./coverage.xml token: ${{ secrets.CODECOV_TOKEN }} - # TODO: evalml_integration_tests: - # TODO: fasttext_integration_tests - - # TODO(aarnphm): add Trax/Flax support - fastai_integration_tests: needs: diff if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.fastai == 'true') || github.event_name == 'push' }} @@ -386,80 +201,26 @@ jobs: - name: Install dependencies run: | pip install . + pip install fastai torch torchvision pandas scikit-learn pip install -r requirements/tests-requirements.txt - name: Run tests and generate coverage report - run: make tests-fastai - shell: bash - - - name: Run unit test for fastai - shell: bash run: | - OPTS=(--cov=src/bentoml --cov-config=./pyproject.toml --cov-report=xml:"fastai.unit.xml" --cov-report term-missing:skip-covered) - python -m pytest ./tests/integration/frameworks/test_fastai_unit.py "${OPTS[@]}" || ERR=1 - if [ $ERR -eq 1 ]; then - echo "unit tests for fastai failed!" - exit 1 - fi + OPTS=(--cov-config pyproject.toml --cov src/bentoml --cov-append --framework fastai) + coverage run -m pytest tests/integration/frameworks/test_frameworks.py tests/integration/frameworks/test_fastai_unit.py "${OPTS[@]}" - - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.fastai == 'true' }} - uses: codecov/codecov-action@v3 - with: - directory: ./ - files: ./fastai.xml,./fastai.unit.xml - flags: fastai - verbose: true - token: ${{ secrets.CODECOV_TOKEN }} - - gluon_integration_tests: - needs: diff - if: ${{ false }} - #if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.gluon == 'true') || github.event_name == 'push' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # fetch all tags and branches - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Get pip cache dir - id: cache-dir - run: | - echo ::set-output name=dir::$(pip cache dir) - - - name: Cache pip dependencies - uses: actions/cache@v3 - id: cache-pip - with: - path: ${{ steps.cache-dir.outputs.dir }} - key: ${{ runner.os }}-tests-${{ hashFiles('requirements/tests-requirements.txt') }} - - - name: Install dependencies - run: | - pip install . - pip install -r requirements/tests-requirements.txt - - - name: Run tests and generate coverage report - run: make tests-gluon - shell: bash + - name: Generate coverage + run: coverage xml - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.gluon == 'true' }} uses: codecov/codecov-action@v3 with: - directory: ./ - files: ./gluon.xml - flags: gluon - verbose: true + files: ./coverage.xml + token: ${{ secrets.CODECOV_TOKEN }} - h2o_integration_tests: + keras_integration_tests: needs: diff - #if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.h2o == 'true') || github.event_name == 'push' }} - if: ${{ false }} + if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.keras == 'true') || github.event_name == 'push' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -468,7 +229,7 @@ jobs: - name: Setup python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: 3.7.10 - name: Get pip cache dir id: cache-dir @@ -485,106 +246,21 @@ jobs: - name: Install dependencies run: | pip install . + pip install keras "tensorflow>=2.7.3" pip install -r requirements/tests-requirements.txt - name: Run tests and generate coverage report - run: make tests-h2o - shell: bash - - - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.h2o == 'true' }} - uses: codecov/codecov-action@v3 - with: - directory: ./ - files: ./h2o.xml - flags: h2o - verbose: true - - # keras_with_tf1_integration_tests: - # needs: diff - # if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.keras_tf1 == 'true') || github.event_name == 'push' }} - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v3 - # with: - # fetch-depth: 0 # fetch all tags and branches - # - name: Setup python - # uses: actions/setup-python@v4 - # with: - # python-version: 3.7.10 - - # - name: Get pip cache dir - # id: cache-dir - # run: | - # echo ::set-output name=dir::$(pip cache dir) - - # - name: Cache pip dependencies - # uses: actions/cache@v3 - # id: cache-pip - # with: - # path: ${{ steps.cache-dir.outputs.dir }} - # key: ${{ runner.os }}-tests-${{ hashFiles('requirements/tests-requirements.txt') }} - - # - name: Install dependencies - # run: | - # pip install . - # pip install -r requirements/tests-requirements.txt - - # - name: Run tests and generate coverage report - # run: make tests-keras_tf1 - # shell: bash - - # - name: Upload test coverage to Codecov - # if: ${{ needs.diff.outputs.keras_tf1 == 'true' }} - # uses: codecov/codecov-action@v3 - # with: - # directory: ./ - # files: ./keras_tf1.xml - # flags: keras - # verbose: true - - keras_with_tf2_integration_tests: - needs: diff - if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.keras_tf2 == 'true') || github.event_name == 'push' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # fetch all tags and branches - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: 3.7.10 - - - name: Get pip cache dir - id: cache-dir - run: | - echo ::set-output name=dir::$(pip cache dir) - - - name: Cache pip dependencies - uses: actions/cache@v3 - id: cache-pip - with: - path: ${{ steps.cache-dir.outputs.dir }} - key: ${{ runner.os }}-tests-${{ hashFiles('requirements/tests-requirements.txt') }} - - - name: Install dependencies run: | - pip install . - pip install -r requirements/tests-requirements.txt + OPTS=(--cov-config pyproject.toml --cov src/bentoml --cov-append --framework keras) + coverage run -m pytest tests/integration/frameworks/test_frameworks.py "${OPTS[@]}" - - name: Run tests and generate coverage report - run: make tests-keras_tf2 - shell: bash + - name: Generate coverage + run: coverage xml - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.keras_tf2 == 'true' }} uses: codecov/codecov-action@v3 with: - directory: ./ - files: ./keras_tf2.xml - flags: keras - verbose: true + files: ./coverage.xml token: ${{ secrets.CODECOV_TOKEN }} lightgbm_integration_tests: @@ -615,20 +291,21 @@ jobs: - name: Install dependencies run: | pip install . + pip install lightgbm pip install -r requirements/tests-requirements.txt - name: Run tests and generate coverage report - run: make tests-lightgbm - shell: bash + run: | + OPTS=(--cov-config pyproject.toml --cov src/bentoml --cov-append --framework lightgbm) + coverage run -m pytest tests/integration/frameworks/test_frameworks.py "${OPTS[@]}" + + - name: Generate coverage + run: coverage xml - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.lightgbm == 'true' }} uses: codecov/codecov-action@v3 with: - directory: ./ - files: ./lightgbm.xml - flags: lightgbm - verbose: true + files: ./coverage.xml token: ${{ secrets.CODECOV_TOKEN }} mlflow_integration_tests: @@ -659,20 +336,21 @@ jobs: - name: Install dependencies run: | pip install . + pip install mlflow scikit-learn pip install -r requirements/tests-requirements.txt - name: Run tests and generate coverage report - run: make tests-mlflow - shell: bash + run: | + OPTS=(--cov-config pyproject.toml --cov src/bentoml --cov-append) + coverage run -m pytest tests/integration/frameworks/mlflow "${OPTS[@]}" + + - name: Generate coverage + run: coverage xml - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.mlflow == 'true' }} uses: codecov/codecov-action@v3 with: - directory: ./ - files: ./mlflow.xml - flags: mlflow - verbose: true + files: ./coverage.xml token: ${{ secrets.CODECOV_TOKEN }} onnx_integration_tests: @@ -703,67 +381,22 @@ jobs: - name: Install dependencies run: | pip install . + pip install onnx onnxruntime pip install -r requirements/tests-requirements.txt - name: Run tests and generate coverage report - run: make tests-onnx - shell: bash - - - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.onnx == 'true' }} - uses: codecov/codecov-action@v3 - with: - directory: ./ - files: ./onnx.xml - flags: onnx - verbose: true - token: ${{ secrets.CODECOV_TOKEN }} - - # TODO:onnxmlir_integration_tests - - paddle_integration_tests: - needs: diff - #if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.paddle == 'true') || github.event_name == 'push' }} - if: ${{ false }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # fetch all tags and branches - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Get pip cache dir - id: cache-dir run: | - echo ::set-output name=dir::$(pip cache dir) + OPTS=(--cov-config pyproject.toml --cov src/bentoml --cov-append --framework onnx) + coverage run -m pytest tests/integration/frameworks/test_frameworks.py "${OPTS[@]}" - - name: Cache pip dependencies - uses: actions/cache@v3 - id: cache-pip - with: - path: ${{ steps.cache-dir.outputs.dir }} - key: ${{ runner.os }}-tests-${{ hashFiles('requirements/tests-requirements.txt') }} - - - name: Install dependencies - run: | - pip install . - pip install -r requirements/tests-requirements.txt - - - name: Run tests and generate coverage report - run: make tests-paddle - shell: bash + - name: Generate coverage + run: coverage xml - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.paddle == 'true' }} uses: codecov/codecov-action@v3 with: - directory: ./ - files: ./paddle.xml - flags: paddle - verbose: true + files: ./coverage.xml + token: ${{ secrets.CODECOV_TOKEN }} picklable_model_integration_tests: needs: diff @@ -796,64 +429,18 @@ jobs: pip install -r requirements/tests-requirements.txt - name: Run tests and generate coverage report - run: make tests-picklable_model - shell: bash - - - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.paddle == 'true' }} - uses: codecov/codecov-action@v3 - with: - directory: ./ - files: ./picklable_model.xml - flags: picklable_model - verbose: true - token: ${{ secrets.CODECOV_TOKEN }} - - pycaret_integration_tests: - needs: diff - #if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.pycaret == 'true') || github.event_name == 'push' }} - if: ${{ false }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # fetch all tags and branches - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Get pip cache dir - id: cache-dir - run: | - echo ::set-output name=dir::$(pip cache dir) - - - name: Cache pip dependencies - uses: actions/cache@v3 - id: cache-pip - with: - path: ${{ steps.cache-dir.outputs.dir }} - key: ${{ runner.os }}-tests-${{ hashFiles('requirements/tests-requirements.txt') }} - - - name: Install dependencies run: | - pip install . - pip install -r requirements/tests-requirements.txt + OPTS=(--cov-config pyproject.toml --cov src/bentoml --cov-append --framework picklable_model) + coverage run -m pytest tests/integration/frameworks/test_frameworks.py "${OPTS[@]}" - - name: Run tests and generate coverage report - run: make tests-pycaret - shell: bash + - name: Generate coverage + run: coverage xml - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.pycaret == 'true' }} uses: codecov/codecov-action@v3 with: - directory: ./ - files: ./pycaret.xml - flags: pycaret - verbose: true - - # TODO: pyspark_mllib_integration_tests + files: ./coverage.xml + token: ${{ secrets.CODECOV_TOKEN }} pytorch_integration_tests: needs: diff @@ -883,30 +470,21 @@ jobs: - name: Install dependencies run: | pip install . + pip install torch torchvision pip install -r requirements/tests-requirements.txt - name: Run tests and generate coverage report - run: make tests-pytorch - shell: bash - - - name: Run unit test for pytorch - shell: bash run: | - OPTS=(--cov=src/bentoml --cov-config=./pyproject.toml --cov-report=xml:"pytorch.unit.xml" --cov-report term-missing:skip-covered) - python -m pytest ./tests/integration/frameworks/test_pytorch_unit.py "${OPTS[@]}" || ERR=1 - if [ $ERR -eq 1 ]; then - echo "unit tests for pytorch failed!" - exit 1 - fi + OPTS=(--cov-config pyproject.toml --cov src/bentoml --cov-append --framework pytorch) + coverage run -m pytest tests/integration/frameworks/test_frameworks.py tests/integration/frameworks/test_pytorch_unit.py "${OPTS[@]}" + + - name: Generate coverage + run: coverage xml - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.pytorch == 'true' }} uses: codecov/codecov-action@v3 with: - directory: ./ - files: ./pytorch.xml,./pytorch.unit.xml - flags: pytorch - verbose: true + files: ./coverage.xml token: ${{ secrets.CODECOV_TOKEN }} pytorch_lightning_integration_tests: @@ -937,20 +515,21 @@ jobs: - name: Install dependencies run: | pip install . + pip install torch torchvision lightning pip install -r requirements/tests-requirements.txt - name: Run tests and generate coverage report - run: make tests-pytorch_lightning - shell: bash + run: | + OPTS=(--cov-config pyproject.toml --cov src/bentoml --cov-append --framework pytorch_lightning) + coverage run -m pytest tests/integration/frameworks/test_frameworks.py "${OPTS[@]}" + + - name: Generate coverage + run: coverage xml - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.pytorch_lightning == 'true' }} uses: codecov/codecov-action@v3 with: - directory: ./ - files: ./pytorch_lightning.xml - flags: pytorch_lightning - verbose: true + files: ./coverage.xml token: ${{ secrets.CODECOV_TOKEN }} torchscript_integration_tests: @@ -981,20 +560,21 @@ jobs: - name: Install dependencies run: | pip install . + pip install torch torchvision pip install -r requirements/tests-requirements.txt - name: Run tests and generate coverage report - run: make tests-torchscript - shell: bash + run: | + OPTS=(--cov-config pyproject.toml --cov src/bentoml --cov-append --framework torchscript) + coverage run -m pytest tests/integration/frameworks/test_frameworks.py "${OPTS[@]}" + + - name: Generate coverage + run: coverage xml - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.torchscript == 'true' }} uses: codecov/codecov-action@v3 with: - directory: ./ - files: ./torchscript.xml - flags: torchscript - verbose: true + files: ./coverage.xml token: ${{ secrets.CODECOV_TOKEN }} sklearn_integration_tests: @@ -1025,70 +605,26 @@ jobs: - name: Install dependencies run: | pip install . + pip install joblib scikit-learn pip install -r requirements/tests-requirements.txt - name: Run tests and generate coverage report - run: make tests-sklearn - shell: bash - - - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.sklearn == 'true' }} - uses: codecov/codecov-action@v3 - with: - directory: ./ - files: ./sklearn.xml - flags: sklearn - verbose: true - token: ${{ secrets.CODECOV_TOKEN }} - - spacy_integration_tests: - needs: diff - #if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.spacy == 'true') || github.event_name == 'push' }} - if: ${{ false }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # fetch all tags and branches - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Get pip cache dir - id: cache-dir - run: | - echo ::set-output name=dir::$(pip cache dir) - - - name: Cache pip dependencies - uses: actions/cache@v3 - id: cache-pip - with: - path: ${{ steps.cache-dir.outputs.dir }} - key: ${{ runner.os }}-tests-${{ hashFiles('requirements/tests-requirements.txt') }} - - - name: Install dependencies run: | - pip install . - pip install -r requirements/tests-requirements.txt + OPTS=(--cov-config pyproject.toml --cov src/bentoml --cov-append --framework sklearn) + coverage run -m pytest tests/integration/frameworks/test_frameworks.py "${OPTS[@]}" - - name: Run tests and generate coverage report - run: make tests-spacy - shell: bash + - name: Generate coverage + run: coverage xml - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.spacy == 'true' }} uses: codecov/codecov-action@v3 with: - directory: ./ - files: ./spacy.xml - flags: spacy - verbose: true + files: ./coverage.xml + token: ${{ secrets.CODECOV_TOKEN }} - statsmodels_integration_tests: + tensorflow_integration_tests: needs: diff - #if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.statsmodels == 'true') || github.event_name == 'push' }} - if: ${{ false }} + if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.tensorflow == 'true') || github.event_name == 'push' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -1114,127 +650,26 @@ jobs: - name: Install dependencies run: | pip install . + pip install "tensorflow>=2.7.3" pip install -r requirements/tests-requirements.txt - name: Run tests and generate coverage report - run: make tests-statsmodels - shell: bash - - - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.statsmodels == 'true' }} - uses: codecov/codecov-action@v3 - with: - directory: ./ - files: ./statsmodels.xml - flags: statsmodels - verbose: true - - tensorflow_v1_integration_tests: - needs: diff - #if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.tf1 == 'true') || github.event_name == 'push' }} - if: ${{ false }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # fetch all tags and branches - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: 3.7 - - - name: Get pip cache dir - id: cache-dir - run: | - echo ::set-output name=dir::$(pip cache dir) - - - name: Cache pip dependencies - uses: actions/cache@v3 - id: cache-pip - with: - path: ${{ steps.cache-dir.outputs.dir }} - key: ${{ runner.os }}-tests-${{ hashFiles('requirements/tests-requirements.txt') }} - - - name: Install dependencies - run: | - pip install . - pip install -r requirements/tests-requirements.txt - - - name: Run tests and generate coverage report - run: make tests-tf1 - shell: bash - - - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.tf1 == 'true' }} - uses: codecov/codecov-action@v3 - with: - directory: ./ - files: ./tf1.xml - flags: tensorflow - verbose: true - - tensorflow_v2_integration_tests: - needs: diff - if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.tf2 == 'true') || github.event_name == 'push' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # fetch all tags and branches - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Get pip cache dir - id: cache-dir - run: | - echo ::set-output name=dir::$(pip cache dir) - - - name: Cache pip dependencies - uses: actions/cache@v3 - id: cache-pip - with: - path: ${{ steps.cache-dir.outputs.dir }} - key: ${{ runner.os }}-tests-${{ hashFiles('requirements/tests-requirements.txt') }} - - - name: Install dependencies run: | - pip install . - pip install -r requirements/tests-requirements.txt + OPTS=(--cov-config pyproject.toml --cov src/bentoml --cov-append --framework tensorflow) + coverage run -m pytest tests/integration/frameworks/test_frameworks.py tests/integration/frameworks/test_tensorflow_unit.py "${OPTS[@]}" - - name: Run tests and generate coverage report - run: make tests-tensorflow_v2 - shell: bash + - name: Generate coverage + run: coverage xml - - name: Run unit test for tensorflow_v2 - shell: bash + - name: Run tests for no eager execution run: | - OPTS=(--cov=src/bentoml --cov-config=./pyproject.toml --cov-report=xml:"tf2.unit.xml" --cov-report term-missing:skip-covered) - python -m pytest ./tests/integration/frameworks/test_tensorflow_v2_unit.py "${OPTS[@]}" || ERR=1 - if [ $ERR -eq 1 ]; then - echo "unit tests for tensorflow_v2 failed!" - exit 1 - fi - - - name: Run tensorflow_v2 without eager execution - shell: bash - run: | - OPTS=(--cov=src/bentoml --cov-config=./pyproject.toml --cov-report=xml:"tf2.unit.no_eager.xml" --cov-report term-missing:skip-covered --disable-tf-eager-execution) - python -m pytest ./tests/integration/frameworks/test_tensorflow_v2_unit.py "${OPTS[@]}" || ERR=1 - if [ $ERR -eq 1 ]; then - echo "unit tests for tensorflow_v2 failed!" - exit 1 - fi + OPTS=(--cov-config pyproject.toml --cov src/bentoml --cov-append --disable-tf-eager-execution --cov-report=xml:no_eager_execution.xml) + coverage run -m pytest tests/integration/frameworks/test_tensorflow_unit.py "${OPTS[@]}" - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.tf2 == 'true' }} uses: codecov/codecov-action@v3 with: - directory: ./ - files: ./tf2.xml,./tf2.unit.xml,./tf2.unit.no_eager.xml - flags: tensorflow - verbose: true + files: ./coverage.xml, ./no_eager_execution.xml token: ${{ secrets.CODECOV_TOKEN }} transformers_integration_tests: @@ -1265,30 +700,21 @@ jobs: - name: Install dependencies run: | pip install . + pip install tensorflow tensorflow_hub transformers jax jaxlib flax torch pip install -r requirements/tests-requirements.txt - name: Run tests and generate coverage report - run: make tests-transformers - shell: bash - - - name: Run unit test for transformers - shell: bash run: | - OPTS=(--cov=src/bentoml --cov-config=./pyproject.toml --cov-report=xml:"transformers.unit.xml" --cov-report term-missing:skip-covered) - python -m pytest ./tests/integration/frameworks/test_transformers_unit.py "${OPTS[@]}" || ERR=1 - if [ $ERR -eq 1 ]; then - echo "unit tests for transformers failed!" - exit 1 - fi + OPTS=(--cov-config pyproject.toml --cov src/bentoml --cov-append --framework transformers) + coverage run -m pytest tests/integration/frameworks/test_frameworks.py tests/integration/frameworks/test_transformers_unit.py "${OPTS[@]}" + + - name: Generate coverage + run: coverage xml - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.transformers == 'true' }} uses: codecov/codecov-action@v3 with: - directory: ./ - files: ./transformers.xml - flags: transformers - verbose: true + files: ./coverage.xml, ./no_eager_execution.xml token: ${{ secrets.CODECOV_TOKEN }} xgboost_integration_tests: @@ -1319,20 +745,21 @@ jobs: - name: Install dependencies run: | pip install . + pip install xgboost pip install -r requirements/tests-requirements.txt - name: Run tests and generate coverage report - run: make tests-xgboost - shell: bash + run: | + OPTS=(--cov-config pyproject.toml --cov src/bentoml --cov-append --framework xgboost) + coverage run -m pytest tests/integration/frameworks/test_frameworks.py "${OPTS[@]}" + + - name: Generate coverage + run: coverage xml - name: Upload test coverage to Codecov - if: ${{ needs.diff.outputs.xgboost == 'true' }} uses: codecov/codecov-action@v3 with: - directory: ./ - files: ./xgboost.xml - flags: xgboost - verbose: true + files: ./coverage.xml token: ${{ secrets.CODECOV_TOKEN }} concurrency: diff --git a/.gitignore b/.gitignore index 22cb9b07b33..8e4fdb9356c 100644 --- a/.gitignore +++ b/.gitignore @@ -149,3 +149,5 @@ Package.resolved **/*/php/GPBMetadata composer.lock vendor +*.pkl +*.ckpt diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 84c1cf1fcdd..b53185b663d 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -300,40 +300,23 @@ Run all unit tests directly with pytest: ```bash # GIT_ROOT=$(git rev-parse --show-toplevel) -pytest tests/unit --cov=bentoml --cov-config="$GIT_ROOT"/pyproject.toml -``` - -Run all unit tests via `./scripts/ci/run_tests.sh`: - -```bash -./scripts/ci/run_tests.sh unit - -# Or on UNIX-based system -make tests-unit +pytest tests/unit ``` ### Integration tests -Run given tests after defining a target under `scripts/ci/config.yml` with `run_tests.sh`: +Write a general framework tests under `./tests/integration/frameworks/models`, and the +run the following command ```bash -# example: run Keras TF1 integration tests -./scripts/ci/run_tests.sh keras_tf1 +pytest tests/integration/frameworks/test_frameworks.py --framework pytorch ``` ### E2E tests ```bash # example: run e2e tests to check for http general features -./scripts/ci/run_tests.sh http_server -``` - -### Running the whole suite - -To run the whole test suite, minus frameworks integration, you can use: - -```bash -make tests-suite +pytest tests/e2e/bento_server_grpc ``` ### Adding new test suite @@ -342,75 +325,7 @@ If you are adding new ML framework support, it is recommended that you also add We recommend using [`nektos/act`](https://github.com/nektos/act) to run and test Actions locally. -The following tests script [run_tests.sh](./scripts/ci/run_tests.sh) can be used to run tests locally. - -```bash -./scripts/ci/run_tests.sh -h -Running unit/integration tests with pytest and generate coverage reports. Make sure that given testcases is defined under ./scripts/ci/config.yml. - -Usage: - ./scripts/ci/run_tests.sh [-h|--help] [-v|--verbose] - -Flags: - -h, --help show this message - -v, --verbose set verbose scripts - - -If `pytest_additional_arguments` is given, the additional arguments will be passed to all of the tests run by the tests script. - -Example: - $ ./scripts/ci/run_tests.sh pytorch --run-gpus-tests --capture=tee-sys -``` - -All tests are then defined under [config.yml](./scripts/ci/config.yml) where each field follows the following format: - -```yaml -: &tmpl - root_test_dir: "tests/integration/frameworks" - is_dir: false - override_name_or_path: - dependencies: [] - external_scripts: - type_tests: "integration" -``` - -By default, each of our frameworks tests files with the format: `test__impl.py`. If `is_dir` set to `true` we will try to match the given `` under `root_test_dir` to run tests from. - -| Keys | Type | Defintions | -| ----------------------- | --------------------------------------- | ------------------------------------------------------------------------------------------------ | -| `root_test_dir` | `` | root directory to run a given tests | -| `is_dir` | `` | whether `target` is a directory instead of a file | -| `override_name_or_path` | `` | optional way to override a tests file name if doesn't match our convention | -| `dependencies` | `` | define additional dependencies required to run the tests, accepts `requirements.txt` format | -| `external_scripts` | `` | optional shell scripts that can be run on top of `./scripts/ci/run_tests.sh` for given testsuite | -| `type_tests` | `` | define type of tests for given `target` | - -When `type_tests` is set to `e2e`, `./scripts/ci/run_tests.sh` will change current directory into the given `root_test_dir`, and will run the testsuite from there. - -The reason why we encourage developers to use the scripts in CI is that under the hood when we use pytest, we will create a custom report for the given tests. This report can then be used as carryforward flags on codecov for consistent reporting. - -Example: - -```yaml -# e2e tests -http: - root_test_dir: "tests/e2e/bento_server_http" - is_dir: true - type_tests: "e2e" - dependencies: - - "Pillow" - -# framework -pytorch_lightning: - <<: *tmpl - dependencies: - - "pytorch-lightning" - - "-f https://download.pytorch.org/whl/torch_stable.html" - - "torch==1.9.0+cpu" - - "torchvision==0.10.0+cpu" -``` - -Refer to [config.yml](./scripts/ci/config.yml) for more examples. +Add a new job for your new framework under [framework.yml](./.github/workflows/frameworks.yml) ## Python tools ecosystem @@ -420,14 +335,6 @@ Currently, BentoML is [PEP518](https://www.python.org/dev/peps/pep-0518/) compat BentoML has moved its benchmark to [`bentoml/benchmark`](https://github.com/bentoml/benchmark). -## Optional: git hooks - -BentoML also provides git hooks that developers can install with: - -```bash -make hooks -``` - ## Creating Pull Requests on GitHub Push changes to your fork and follow [this diff --git a/Makefile b/Makefile index e7824a40033..bf70344b52f 100644 --- a/Makefile +++ b/Makefile @@ -11,17 +11,28 @@ help: ## Show all Makefile targets .PHONY: format format-proto lint lint-proto type style clean format: ## Running code formatter: black and isort - @./scripts/tools/formatter.sh + @echo "(black) Formatting codebase..." + @black --config pyproject.toml src tests docs examples + @echo "(black) Formatting stubs..." + @find src -name "*.pyi" ! -name "*_pb2*" -exec black --pyi --config pyproject.toml {} \; + @echo "(isort) Reordering imports..." + isort . format-proto: ## Running proto formatter: buf @echo "Formatting proto files..." - docker run --init --rm --volume $(GIT_ROOT)/src:/workspace --workdir /workspace bufbuild/buf format --config "/workspace/bentoml/grpc/buf.yaml" -w --path "bentoml/grpc" + docker run --init --rm --volume $(GIT_ROOT)/src:/workspace --workdir /workspace bufbuild/buf format --config "/workspace/bentoml/grpc/buf.yaml" -w bentoml/grpc lint: ## Running lint checker: pylint - @./scripts/tools/linter.sh + @echo "(pylint) Linting bentoml..." + @pylint --rcfile=pyproject.toml --fail-under 9.5 src + @echo "(pylint) Linting examples..." + @pylint --rcfile=pyproject.toml --fail-under 9.5 examples + @echo "(pylint) Linting tests..." + @pylint --rcfile=pyproject.toml --fail-under 9.5 tests lint-proto: ## Running proto lint checker: buf @echo "Linting proto files..." - docker run --init --rm --volume $(GIT_ROOT)/src:/workspace --workdir /workspace bufbuild/buf lint --config "/workspace/bentoml/grpc/buf.yaml" --error-format msvs --path "bentoml/grpc" + docker run --init --rm --volume $(GIT_ROOT)/src:/workspace --workdir /workspace bufbuild/buf lint --config "/workspace/bentoml/grpc/buf.yaml" --error-format msvs bentoml/grpc type: ## Running type checker: pyright - @./scripts/tools/type_checker.sh + @echo "(pyright) Typechecking codebase..." + @pyright -p src -w style: format lint format-proto lint-proto ## Running formatter and linter clean: ## Clean all generated files @echo "Cleaning all generated files..." @@ -29,47 +40,6 @@ clean: ## Clean all generated files @cd $(GIT_ROOT) || exit 1 @find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete - -ci-%: - $(eval style := $(subst ci-, ,$@)) - @./scripts/ci/style/$(style)_check.sh - -.PHONY: ci-format -ci-format: ci-black ci-isort ## Running format check in CI: black, isort - -.PHONY: ci-lint -ci-lint: ci-pylint ## Running lint check in CI: pylint - -.PHONY: tests-suite -tests-suite: tests-unit tests-http_server tests-grpc_server ## Running BentoML tests suite (unit, e2e, integration) - -tests-%: - $(eval type :=$(subst tests-, , $@)) - $(eval RUN_ARGS:=$(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))) - $(eval __positional:=$(foreach t, $(RUN_ARGS), -$(t))) -ifeq ($(USE_VERBOSE),true) - ./scripts/ci/run_tests.sh -v $(type) $(__positional) -else ifeq ($(USE_GPU),true) - ./scripts/ci/run_tests.sh -v $(type) --run-gpu-tests $(__positional) -else ifeq ($(USE_GPRC),true) - ./scripts/ci/run_tests.sh -v $(type) --run-gprc-tests $(__positional) -else - ./scripts/ci/run_tests.sh $(type) $(__positional) -endif - - -install-local: ## Install BentoML in editable mode - @pip install --editable . -install-dev-deps: ## Install all dev dependencies - @echo Installing dev dependencies... - @pip install -r requirements/dev-requirements.txt -install-tests-deps: ## Install all tests dependencies - @echo Installing tests dependencies... - @pip install -r requirements/tests-requirements.txt -install-docs-deps: ## Install documentation dependencies - @echo Installing docs dependencies... - @pip install -r requirements/docs-requirements.txt - # Docs watch-docs: install-docs-deps ## Build and watch documentation sphinx-autobuild docs/source docs/build/html --watch $(GIT_ROOT)/bentoml --ignore "bazel-*" diff --git a/README.md b/README.md index 5e14e527443..ca819d6d129 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,18 @@ [](https://github.com/bentoml/BentoML)
-# The Unified Model Serving Framework [![Tweet](https://img.shields.io/twitter/url/http/shields.io.svg?style=social)](https://twitter.com/intent/tweet?text=BentoML:%20The%20Unified%20Model%20Serving%20Framework%20&url=https://github.com/bentoml&via=bentomlai&hashtags=mlops,bentoml) +# The Unified Model Serving Framework [![Tweet](https://img.shields.io/twitter/url/http/shields.io.svg?style=social)](https://twitter.com/intent/tweet?text=BentoML:%20The%20Unified%20Model%20Serving%20Framework%20&url=https://github.com/bentoml&via=bentomlai&hashtags=mlops,bentoml) [![pypi_status](https://img.shields.io/pypi/v/bentoml.svg)](https://pypi.org/project/BentoML) [![downloads](https://pepy.tech/badge/bentoml)](https://pepy.tech/project/bentoml) [![actions_status](https://github.com/bentoml/bentoml/workflows/CI/badge.svg)](https://github.com/bentoml/bentoml/actions) [![documentation_status](https://readthedocs.org/projects/bentoml/badge/?version=latest)](https://docs.bentoml.org/) [![join_slack](https://badgen.net/badge/Join/BentoML%20Slack/cyan?icon=slack)](https://join.slack.bentoml.org) +[![codecov](https://codecov.io/gh/bentoml/BentoML/branch/main/graph/badge.svg?token=GMzgXdpE5b)](https://codecov.io/gh/bentoml/BentoML) BentoML makes it easy to create Machine Learning services that are ready to deploy and scale. -## Getting Started ## +## Getting Started - [Documentation](https://docs.bentoml.org/) - Overview of the BentoML docs and related resources - [Tutorial: Intro to BentoML](https://docs.bentoml.org/en/latest/tutorial.html) - Learn by doing! In under 10 minutes, you'll serve a model via REST API and generate a docker image for deployment. @@ -23,31 +24,34 @@ BentoML makes it easy to create Machine Learning services that are ready to depl --- - ## Highlights 🍭 Unified Model Serving API + - Framework-agnostic model packaging for Tensorflow, PyTorch, XGBoost, Scikit-Learn, ONNX, and [many more](https://docs.bentoml.org/en/latest/frameworks/index.html)! - Write **custom Python code** alongside model inference for pre/post-processing and business logic - Apply the **same code** for online(REST API or gRPC), offline batch, and streaming inference - Simple abstractions for building **multi-model inference** pipelines or graphs 🚂 **Standardized process** for a frictionless transition to production + - Build [Bento](https://docs.bentoml.org/en/latest/concepts/bento.html) as the standard deployable artifact for ML services - Automatically **generate docker images** with the desired dependencies - Easy CUDA setup for inference with GPU - Rich integration with the MLOps ecosystem, including Kubeflow, Airflow, MLFlow, Triton -🏹 ***Scalable*** with powerful performance optimizations +🏹 **_Scalable_** with powerful performance optimizations + - [Adaptive batching](https://docs.bentoml.org/en/latest/guides/batching.html) dynamically groups inference requests on server-side optimal performance - [Runner](https://docs.bentoml.org/en/latest/concepts/runner.html) abstraction scales model inference separately from your custom code - [Maximize your GPU](https://docs.bentoml.org/en/latest/guides/gpu.html) and multi-core CPU utilization with automatic provisioning 🎯 Deploy anywhere in a **DevOps-friendly** way + - Streamline production deployment workflow via: - - [☁️ BentoML Cloud](https://bentoml.com/): the fastest way to deploy your bento, simple and at scale - - [🦄️ Yatai](https://github.com/bentoml/yatai): Model Deployment at scale on Kubernetes - - [🚀 bentoctl](https://github.com/bentoml/bentoctl): Fast model deployment on AWS SageMaker, Lambda, ECE, GCP, Azure, Heroku, and more! + - [☁️ BentoML Cloud](https://bentoml.com/): the fastest way to deploy your bento, simple and at scale + - [🦄️ Yatai](https://github.com/bentoml/yatai): Model Deployment at scale on Kubernetes + - [🚀 bentoctl](https://github.com/bentoml/bentoctl): Fast model deployment on AWS SageMaker, Lambda, ECE, GCP, Azure, Heroku, and more! - Run offline batch inference jobs with Spark or Dask - Built-in support for Prometheus metrics and OpenTelemetry - Flexible APIs for advanced CI/CD workflows @@ -93,7 +97,7 @@ Create a `bentofile.yaml` build file for your ML service: ```yaml service: "service:svc" include: -- "*.py" + - "*.py" python: packages: - numpy @@ -102,11 +106,13 @@ python: ``` Now, run the prediction service: + ```bash bentoml serve ``` Sent a prediction request: + ```bash curl -F 'image=@samples/1.png' http://127.0.0.1:3000/predict_image ``` @@ -146,6 +152,7 @@ There are many ways to contribute to the project: - Learn more in the [contributing guide](https://github.com/bentoml/BentoML/blob/main/CONTRIBUTING.md). ### Contributors + Thanks to all of our amazing contributors! @@ -154,10 +161,10 @@ Thanks to all of our amazing contributors! --- -### Usage Reporting ### +### Usage Reporting BentoML collects usage data that helps our team to improve the product. -Only BentoML's internal API calls are being reported. We strip out as much potentially +Only BentoML's internal API calls are being reported. We strip out as much potentially sensitive information as possible, and we will never collect user code, model data, model names, or stack traces. Here's the [code](./src/bentoml/_internal/utils/analytics/usage_stats.py) for usage tracking. You can opt-out of usage tracking by the `--do-not-track` CLI option: @@ -167,12 +174,14 @@ bentoml [command] --do-not-track ``` Or by setting environment variable `BENTOML_DO_NOT_TRACK=True`: + ```bash export BENTOML_DO_NOT_TRACK=True ``` + --- -### License ### +### License [Apache License 2.0](https://github.com/bentoml/BentoML/blob/main/LICENSE) diff --git a/codecov.yml b/codecov.yml index e10567af2c9..92dc6e61804 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,5 +1,6 @@ github_checks: annotations: false + comment: layout: "reach, diff, files" behavior: default @@ -9,339 +10,10 @@ comment: - "main" show_carryforward_flags: true -ignore: - - "src/bentoml/types.py" - - "src/bentoml/__main__.py" - - "src/bentoml/_internal/types.py" - - "src/bentoml/io.py" - - "src/bentoml/testing" - - "src/bentoml/_internal/external_typing" - - "src/bentoml/grpc/v1alpha1" - - "tests" - - "typings" - coverage: precision: 2 round: down range: "80...100" status: - default_rules: - flag_coverage_not_uploaded_behavior: exclude patch: off - project: - default: - target: auto - threshold: 10% - paths: - - "src/bentoml/*.py" - - "src/bentoml/_internal/**/*" - flags: - - catboost - - detectron - - easyocr - - evalml - - fastai - - fasttext - - flax - - gluon - - h2o - - keras - - lightgbm - - mlflow - - onnx - - onnxmlir - - paddle - - picklable_model - - pycaret - - pyspark - - pytorch - - pytorch_lightning - - sklearn - - spacy - - statsmodels - - tensorflow - - tensorflow_v1 - - torchscript - - transformers - - xgboost - - unit-tests - - e2e-tests-http - - e2e-tests-grpc - catboost: - target: auto - threshold: 10% - flags: - - catboost - detectron: - target: auto - threshold: 10% - flags: - - detectron - easyocr: - target: auto - threshold: 10% - flags: - - easyocr - evalml: - target: auto - threshold: 10% - flags: - - evalml - fastai: - target: auto - threshold: 10% - flags: - - fastai - fasttext: - target: auto - threshold: 10% - flags: - - fasttext - flax: - target: auto - threshold: 10% - flags: - - flax - gluon: - target: auto - threshold: 10% - flags: - - gluon - h2o: - target: auto - threshold: 10% - flags: - - h2o - keras: - target: auto - threshold: 10% - flags: - - keras - lightgbm: - target: auto - threshold: 10% - flags: - - lightgbm - mlflow: - target: auto - threshold: 10% - flags: - - mlflow - onnx: - target: auto - threshold: 10% - flags: - - onnx - onnxmlir: - target: auto - threshold: 10% - flags: - - onnxmlir - paddle: - target: auto - threshold: 10% - flags: - - paddle - picklable_model: - target: auto - threshold: 10% - flags: - - picklable_model - pycaret: - target: auto - threshold: 10% - flags: - - pycaret - pyspark: - target: auto - threshold: 10% - flags: - - pyspark - pytorch: - target: auto - threshold: 10% - flags: - - pytorch - pytorch_lightning: - target: auto - threshold: 10% - flags: - - pytorch_lightning - sklearn: - target: auto - threshold: 10% - flags: - - sklearn - spacy: - target: auto - threshold: 10% - flags: - - spacy - statsmodels: - target: auto - threshold: 10% - flags: - - statsmodels - tensorflow: - target: auto - threshold: 10% - flags: - - tensorflow - tensorflow_v1: - target: auto - threshold: 10% - flags: - - tensorflow_v1 - torchscript: - target: auto - threshold: 10% - flags: - - torchscript - transformers: - target: auto - threshold: 10% - flags: - - transformers - xgboost: - target: auto - threshold: 10% - flags: - - xgboost - e2e: - target: auto - threshold: 10% - flags: - - e2e-tests-http - - e2e-tests-grpc - unit: - target: auto - threshold: 10% - flags: - - unit-tests - -flags: - catboost: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/catboost.py - detectron: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/detectron.py - easyocr: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/easyocr.py - evalml: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/evalml.py - fastai: - carryforward: true - paths: - - src/bentoml/fastai.py - - src/bentoml/_internal/frameworks/fastai.py - fasttext: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/fasttext.py - flax: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/flax.py - gluon: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/gluon.py - h2o: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/h2o.py - keras: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/keras.py - lightgbm: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/lightgbm.py - mlflow: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/mlflow.py - onnx: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/onnx.py - onnxmlir: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/onnxmlir.py - paddle: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/paddle.py - picklable_model: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/picklable.py - pycaret: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/pycaret.py - pyspark: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/pyspark.py - pytorch: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/pytorch.py - pytorch_lightning: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/pytorch_lightning.py - sklearn: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/sklearn.py - spacy: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/spacy.py - statsmodels: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/statsmodels.py - tensorflow: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/tensorflow_v2.py - tensorflow_v1: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/tensorflow_v1.py - transformers: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/transformers.py - xgboost: - carryforward: true - paths: - - src/bentoml/_internal/frameworks/xgboost.py - e2e-tests-http: - carryforward: true - paths: - - "src/bentoml/**/*" - - "src/bentoml/models.py" - e2e-tests-grpc: - carryforward: true - paths: - - "src/bentoml/**/*" - - "src/bentoml/grpc/interceptors/" - - "src/bentoml/grpc/utils/" - unit-tests: - carryforward: true - paths: - - "src/bentoml/**/*" - - "src/bentoml/models.py" + project: off diff --git a/hooks/commit-msg b/hooks/commit-msg deleted file mode 100755 index 367d810c255..00000000000 --- a/hooks/commit-msg +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -if ! head -1 "$1" | grep -qE "^(qa|feat|fix|ci|chore|docs|test|style|refactor|perf|build|revert)(\(.+?\))?: .{1,}$"; then - echo "Aborting commit. Please use ([optional scope]): format for your commit title, where should be one of (qa|feat|fix|ci|chore|docs|test|style|refactor|perf|build|revert)" >&2 - echo 'Some valid examples: "feat: add support for PyTorch" and "fix(cli): add missing arguments of bentoml serve"' - echo "You can read more details about commit message formatting in DEVELOPMENT.md " - exit 1 -fi diff --git a/hooks/pre-commit b/hooks/pre-commit deleted file mode 100755 index 624682fd8e0..00000000000 --- a/hooks/pre-commit +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -LC_ALL=C - -GIT_ROOT="$(git rev-parse --show-toplevel)" - -cd "$GIT_ROOT" || exit - -source "$GIT_ROOT/scripts/ci/helpers.sh" - -# Run format scripts -./scripts/tools/formatter.sh || exit 1 - -INFO "Running yamllint on github workflow..." -find "$GIT_ROOT" -type f -name '*.yml' -exec yamllint -c "$GIT_ROOT/.yamllint.yml" {} \; - -exit 0 diff --git a/pyproject.toml b/pyproject.toml index 13e072a7752..92ea5e40eae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -158,7 +158,7 @@ source = ["src"] [tool.coverage.run] branch = true -source = ["src"] +source = ["src/bentoml/"] omit = [ "src/bentoml/__main__.py", "src/bentoml/io.py", @@ -224,10 +224,7 @@ exclude = ''' ''' [tool.pytest.ini_options] -addopts = [ - "-rfEX", - "-pbentoml.testing.pytest.plugin" -] +addopts = ["-rfEX", "-pbentoml.testing.pytest.plugin"] python_files = ["test_*.py", "*_test.py"] testpaths = ["tests"] @@ -243,93 +240,10 @@ extension-pkg-allow-list = [ "pydantic.schema", ] ignore-paths = [ - "typings", "src/bentoml/_internal/external_typing", "src/bentoml/grpc/v1alpha1", ] -disable = [ - "import-error", - "print-statement", - "parameter-unpacking", - "unpacking-in-except", - "old-raise-syntax", - "backtick", - "raw-checker-failed", - "bad-inline-option", - "locally-disabled", - "file-ignored", - "suppressed-message", - "useless-suppression", - "deprecated-pragma", - "apply-builtin", - "basestring-builtin", - "buffer-builtin", - "cmp-builtin", - "coerce-builtin", - "execfile-builtin", - "file-builtin", - "long-builtin", - "raw_input-builtin", - "reduce-builtin", - "standarderror-builtin", - "coerce-method", - "delslice-method", - "getslice-method", - "setslice-method", - "no-absolute-import", - "old-division", - "dict-iter-method", - "dict-view-method", - "next-method-called", - "metaclass-assignment", - "indexing-exception", - "raising-string", - "reload-builtin", - "oct-method", - "hex-method", - "nonzero-method", - "cmp-method", - "input-builtin", - "round-builtin", - "intern-builtin", - "unichr-builtin", - "map-builtin-not-iterating", - "zip-builtin-not-iterating", - "range-builtin-not-iterating", - "filter-builtin-not-iterating", - "using-cmp-argument", - "exception-message-attribute", - "invalid-str-codec", - "sys-max-int", - "bad-python3-import", - "deprecated-string-function", - "deprecated-str-translate-call", - "deprecated-itertools-function", - "deprecated-types-field", - "next-method-defined", - "dict-items-not-iterating", - "dict-keys-not-iterating", - "dict-values-not-iterating", - "deprecated-operator-function", - "deprecated-urllib-function", - "xreadlines-attribute", - "deprecated-sys-function", - "exception-escape", - "comprehension-escape", - "logging-fstring-interpolation", - "logging-format-interpolation", - "logging-not-lazy", - "C", - "R", - "fixme", - "protected-access", - "no-member", - "unsubscriptable-object", - "raise-missing-from", - "isinstance-second-argument-not-valid-type", - "attribute-defined-outside-init", - "relative-beyond-top-level", -] +disable = ["coerce-builtin", "no-absolute-import", "C", "R"] enable = ["c-extension-no-member"] evaluation = "10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)" msg-template = "{msg_id}:{symbol} [{line:0>3d}:{column:0>2d}] {obj}: {msg}" diff --git a/scripts/ci/config.yml b/scripts/ci/config.yml deleted file mode 100644 index ae1ae266a9e..00000000000 --- a/scripts/ci/config.yml +++ /dev/null @@ -1,255 +0,0 @@ -# NOTE: PUT YOUR PYPI dependencies under `dependencies` -# `dependencies` will pass all given arguments to a `requirements.txt` -# then `./scripts/ci/run_tests.sh` will try to install all given dependencies -# for given tests -# `external_scripts` will be run AFTER installing dependencies, and can be used -# to setup external envvar - -template: &tmpl - root_test_dir: "tests/integration/frameworks" - is_dir: false - override_name_or_path: - dependencies: [] - external_scripts: - type_tests: "integration" - -new_template: &ntmpl - root_test_dir: "tests/integration/frameworks" - is_dir: false - override_name_or_path: "test_frameworks.py" - depedencies: [] - external_scripts: - type_tests: "integration" - -unit: - <<: *tmpl - root_test_dir: "tests/unit" - is_dir: true - type_tests: "unit" - -http_server: - <<: *tmpl - root_test_dir: "tests/e2e/bento_server_http" - is_dir: true - type_tests: "e2e" - dependencies: - - Pillow - - pydantic - - fastapi - -grpc_server: - <<: *tmpl - root_test_dir: "tests/e2e/bento_server_grpc" - is_dir: true - type_tests: "e2e" - dependencies: - - Pillow - - pydantic - -catboost: - <<: *ntmpl - dependencies: - - "catboost" - -detectron2: - <<: *tmpl - dependencies: - - "-f https://download.pytorch.org/whl/torch_stable.html" - - "torch==1.9.0+cpu" - - "torchvision==0.10.0+cpu" - - "-f https://dl.fbaipublicfiles.com/detectron2/wheels/cpu/torch1.9/index.html" - - "detectron2" - -easyocr: - <<: *tmpl - dependencies: - - "easyocr" - - "opencv-python-headless==4.5.4.60" - -evalml: - <<: *tmpl - dependencies: - - "evalml>=0.25.0" - -fastai: - <<: *ntmpl - dependencies: - - fastai - - "-f https://download.pytorch.org/whl/torch_stable.html" - - "torch==1.9.0+cpu" - - "torchvision==0.10.0+cpu" - - pandas - - scikit-learn - -fasttext: - <<: *tmpl - dependencies: - - "fasttext" - -gluon: - <<: *tmpl - dependencies: - - "mxnet" - -h2o: - <<: *tmpl - dependencies: - - "h2o" - -keras_tf1: - <<: *tmpl - override_name_or_path: test_keras_impl.py - dependencies: - - "tensorflow==1.15" - - "h5py==2.10.0" - -keras_tf2: - <<: *ntmpl - dependencies: - - "protobuf<3.20,>=3.9.2" - - "tensorflow==2.7.3" - -lightgbm: - <<: *ntmpl - dependencies: - - "lightgbm" - -# we need to use external_scripts to update scikit-learn here due to some error -# with mismatch scikit-learn from mlflow side -mlflow: - <<: *tmpl - is_dir: true - override_name_or_path: mlflow - dependencies: - - "mlflow" - external_scripts: | - python -m pip install -U scikit-learn - -onnx: - <<: *ntmpl - dependencies: - - "onnx" - - "onnxruntime" - - "skl2onnx" - - "tensorflow" - - "-f https://download.pytorch.org/whl/torch_stable.html" - - "torch==1.9.0+cpu" - -onnxmlir: - <<: *tmpl - dependencies: - - "tensorflow" - - "tf2onnx" - - "pandas" - - "protobuf" - -paddle: - <<: *tmpl - is_dir: true - override_name_or_path: paddle - dependencies: - - "opencv-python==4.5.4.58" - - "paddlepaddle" - - "paddlehub" - - "gast==0.3.3" - external_scripts: | - export SETUPTOOLS_USE_DISTUTILS=stdlib - -picklable_model: - <<: *tmpl - -pycaret: - <<: *tmpl - dependencies: - - "numba==0.53.1" - - "pycaret==2.3.3" - -pyspark: - <<: *tmpl - dependencies: - - "pyspark" - -pytorch: - <<: *ntmpl - dependencies: - - "-f https://download.pytorch.org/whl/torch_stable.html" - - "torch==1.9.0+cpu" - - "torchvision==0.10.0+cpu" - - "psutil" - -sklearn: - <<: *tmpl - -spacy: - <<: *tmpl - is_dir: true - override_name_or_path: spacy - dependencies: - - "spacy==3.1.2" - - "pyyaml" - - "-f https://download.pytorch.org/whl/torch_stable.html" - - "torch==1.9.0+cpu" - - "torchvision==0.10.0+cpu" - - "tensorflow==2.7.3" - external_scripts: | - python -m spacy download en_core_web_sm - python -m pip install "typing_extensions>=3.10" - -statsmodels: - <<: *tmpl - dependencies: - - "statsmodels==0.12.2" - - "scipy==1.7.3" # statsmodels 0.12.2 is using internal APIs of scipy - - "joblib" - -tf1: - <<: *tmpl - override_name_or_path: test_tensorflow_v1_impl.py - dependencies: - - "tensorflow==1.15" - -transformers: - <<: *ntmpl - dependencies: - - "transformers" - - "tensorflow" - - "tensorflow_hub" - - "-f https://download.pytorch.org/whl/torch_stable.html" - - "torch==1.11.0" - - "jax" - - "jaxlib" - - "flax" - - "importlib_metadata" - - "Pillow==8.4.0" - - "typing_extensions==4.2.0" - external_scripts: | - python -m pip install "typing_extensions>=3.10" - -xgboost: - <<: *ntmpl - dependencies: - - "xgboost" - -tensorflow_v2: - <<: *ntmpl - dependencies: - - "tensorflow" - -torchscript: - <<: *ntmpl - dependencies: - - "-f https://download.pytorch.org/whl/torch_stable.html" - - "torch==1.11.0+cpu" - - "torchvision==0.12.0+cpu" - - "protobuf<4.21.0" # https://github.com/PyTorchLightning/pytorch-lightning/issues/13159 - - "psutil" - -pytorch_lightning: - <<: *ntmpl - dependencies: - - "-f https://download.pytorch.org/whl/torch_stable.html" - - "torch==1.11.0+cpu" - - "torchvision==0.12.0+cpu" - - "pytorch_lightning==1.6.3" - - "protobuf<4.21.0" # https://github.com/PyTorchLightning/pytorch-lightning/issues/13159 - - "psutil" diff --git a/scripts/ci/helpers.sh b/scripts/ci/helpers.sh deleted file mode 100644 index 9f43167f73c..00000000000 --- a/scripts/ci/helpers.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[0;33m' -NC='\033[0m' - -PASS() { - echo -e "$GREEN""[PASS]""$NC" "$*" -} - -INFO() { - echo -e "${GREEN}[INFO]${NC}" "$*" -} - -FAIL() { - echo -e "$RED""[FAIL]""$NC" "$*" -} - -WARN() { - echo -e "$YELLOW""[WARN]""$NC" "$*" -} - -set_on_failed_callback() { - set -E - trap "$*" ERR -} - -check_cmd() { - command -v "$1" >/dev/null 2>&1 -} - -need_cmd() { - if ! check_cmd "$1"; then - FAIL "need $1 (command not found)" - exit 1 - fi -} - -set -eo pipefail diff --git a/scripts/ci/run_tests.sh b/scripts/ci/run_tests.sh deleted file mode 100755 index c3dffcbe209..00000000000 --- a/scripts/ci/run_tests.sh +++ /dev/null @@ -1,250 +0,0 @@ -#!/usr/bin/env bash - -# Prerequisite: -# This scripts assumes BentoML and all its test dependencies are already installed: -# -# pip install -e . -# pip install requirements/tests-requirements.txt - -fname=$(basename "$0") -dname=$(dirname "$0") - -# shellcheck disable=SC1091 -source "$dname/helpers.sh" - -set_on_failed_callback "ERR=1" - -GIT_ROOT=$(git rev-parse --show-toplevel) - -declare -a PYTESTARGS -CONFIG_FILE="$dname/config.yml" -REQ_FILE="/tmp/additional-requirements.txt" -SKIP_DEPS=0 -ERR=0 -VERBOSE=0 -ENABLE_XDIST=1 -WORKERS=auto - -cd "$GIT_ROOT" || exit - -run_yq() { - need_cmd yq - yq "$@" -} - -getval() { - run_yq eval "$@" "$CONFIG_FILE" -} - -validate_yaml() { - # validate YAML file - if ! [ -f "$CONFIG_FILE" ]; then - FAIL "$CONFIG_FILE does not exists" - exit 1 - fi - - if ! (run_yq e --exit-status 'tag == "!!map" or tag== "!!seq"' "$CONFIG_FILE" >/dev/null); then - FAIL "Invalid YAML file" - exit 1 - fi -} - -usage() { - need_cmd cat - - cat < - -Flags: - -h, --help show this message - -v, --verbose set verbose scripts - -s, --skip-deps skip install dependencies - -w, --workers number of workers for pytest-xdist - --disable-xdist disable pytest-xdist - - -If pytest_additional_arguments is given, this will be appended to given tests run. - -Example: - $ $dname/$fname pytorch --run-gpu-tests -HEREDOC - exit 2 -} - -parse_args() { - if [ "${#}" -eq 0 ]; then - FAIL "$0 doesn't run without any arguments" - exit 1 - fi - if [ "${1:0:1}" = "-" ]; then - FAIL "First arguments must be a target, not a flag." - exit 1 - fi - - for arg in "$@"; do - case "$arg" in - -h | --help) - usage - ;; - -v | --verbose) - set -x - VERBOSE=1 - shift - ;; - -w | --workers) - shift - WORKERS="$2" - shift - ;; - --disable-xdist) - ENABLE_XDIST=0 - shift - ;; - -s | --skip-deps) - SKIP_DEPS=1 - shift - ;; - *) ;; - - esac - done - PYTESTARGS=("${*:2}") - shift $((OPTIND - 1)) -} - -parse_config() { - target=$@ - test_dir= - is_dir= - override_name_or_path= - external_scripts= - type_tests= - - test_dir=$(getval ".$target.root_test_dir") - is_dir=$(getval ".$target.is_dir") - override_name_or_path=$(getval ".$target.override_name_or_path") - external_scripts=$(getval ".$target.external_scripts") - type_tests=$(getval ".$target.type_tests") - - # processing file name - if [[ "$override_name_or_path" != "" ]]; then - fname="$override_name_or_path" - elif [[ "$is_dir" == "false" ]]; then - fname="test_""$target""_impl.py" - elif [[ "$is_dir" == "true" ]]; then - fname="" - shift - else - fname="$target" - fi - - # processing dependencies - run_yq eval '.'"$target"'.dependencies[]' "$CONFIG_FILE" >"$REQ_FILE" || exit -} - -install_yq() { - set -ex - target_dir="$HOME/.local/bin" - - mkdir -p "$target_dir" - export PATH=$target_dir:$PATH - - YQ_VERSION=4.16.1 - echo "Trying to install yq..." - shell=$(uname | tr '[:upper:]' '[:lower:]') - extensions=".tar.gz" - if [[ "$shell" =~ "mingw64" ]]; then - shell="windows" - extensions=".zip" - fi - - YQ_BINARY=yq_"$shell"_amd64 - YQ_EXTRACT="./$YQ_BINARY" - if [[ "$shell" == "windows" ]]; then - YQ_EXTRACT="$YQ_BINARY.exe" - fi - curl -fsSLO https://github.com/mikefarah/yq/releases/download/v"$YQ_VERSION"/"$YQ_BINARY""$extensions" - echo "tar $YQ_BINARY$extensions and move to /usr/bin/yq..." - if [[ $(uname | tr '[:upper:]' '[:lower:]') =~ "mingw64" ]]; then - unzip -qq "$YQ_BINARY$extensions" -d yq_dir && cd yq_dir - mv "$YQ_EXTRACT" "$target_dir"/yq && cd .. - rm -rf yq_dir - else - tar -zvxf "$YQ_BINARY$extensions" "$YQ_EXTRACT" && mv "$YQ_EXTRACT" "$target_dir"/yq - fi - rm -f ./"$YQ_BINARY""$extensions" -} - -main() { - parse_args "$@" - - need_cmd make - need_cmd curl - need_cmd tr - (need_cmd yq && echo "Using yq via $(which yq)...") || install_yq - - for args in "$@"; do - if [[ "$args" != "-"* ]]; then - argv="$args" - break - else - shift - fi - done - - # validate_yaml - parse_config "$argv" - - OPTS=(--cov-config="$GIT_ROOT/pyproject.toml" --cov-report=xml:"$target.xml") - - if [ -n "${PYTESTARGS[*]}" ]; then - # shellcheck disable=SC2206 - OPTS=(${OPTS[@]} ${PYTESTARGS[@]}) - fi - - if [ "$fname" == "test_frameworks.py" ]; then - OPTS=("--framework" "$target" "${OPTS[@]}") - fi - if [ "$VERBOSE" -eq 1 ]; then - OPTS=("${OPTS[@]}" -vvv) - fi - - if [ "$type_tests" == 'unit' ] && [ "$ENABLE_XDIST" -eq 1 ] && [ "$(uname | tr '[:upper:]' '[:lower:]')" != "win32" ]; then - OPTS=("${OPTS[@]}" --dist loadfile -n "$WORKERS") - fi - - if [ "$SKIP_DEPS" -eq 0 ]; then - # setup tests environment - if [ -f "$REQ_FILE" ]; then - pip install -r "$REQ_FILE" || exit 1 - fi - fi - - if [ -n "$external_scripts" ]; then - eval "$external_scripts" || exit 1 - fi - - if [ "$type_tests" == 'e2e' ]; then - p="$GIT_ROOT/$test_dir" - cd "$p" || exit 1 - path="." - else - path="$GIT_ROOT"/"$test_dir"/"$fname" - fi - - # run pytest - python -m pytest "$path" "${OPTS[@]}" || ERR=1 - - # Return non-zero if pytest failed - if ! test $ERR = 0; then - FAIL "$type_tests tests failed!" - exit 1 - fi - - PASS "$type_tests tests passed!" -} - -main "$@" || exit 1 diff --git a/scripts/ci/style/black_check.sh b/scripts/ci/style/black_check.sh deleted file mode 100755 index 94b5efbe425..00000000000 --- a/scripts/ci/style/black_check.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -GIT_ROOT=$(git rev-parse --show-toplevel) - -cd "$GIT_ROOT" || exit 1 - -source ./scripts/ci/helpers.sh - -echo "Running black format check..." - -set_on_failed_callback "[FAIL] black format check failed" - -if ! (black --check --config "./pyproject.toml" src examples tests typings); then - FAIL "black format check failed" - echo "Make sure to run \`make format\`" - exit 1 -fi - -PASS "black format check passed!" -exit 0 diff --git a/scripts/ci/style/isort_check.sh b/scripts/ci/style/isort_check.sh deleted file mode 100755 index c3a69315d72..00000000000 --- a/scripts/ci/style/isort_check.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -GIT_ROOT=$(git rev-parse --show-toplevel) - -cd "$GIT_ROOT" || exit 1 - -source ./scripts/ci/helpers.sh - -echo "Running isort format check..." - -set_on_failed_callback "[FAIL] isort format check failed" - -if ! (isort --check .); then - FAIL "isort format check failed" - echo "Make sure to run \`make format\`" - exit 1 -fi - -PASS "isort format check passed!" -exit 0 diff --git a/scripts/ci/style/pylint_check.sh b/scripts/ci/style/pylint_check.sh deleted file mode 100755 index f941f2208a3..00000000000 --- a/scripts/ci/style/pylint_check.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -name=$(basename "$0") - -GIT_ROOT=$(git rev-parse --show-toplevel) - -cd "$GIT_ROOT" || exit - -source ./scripts/ci/helpers.sh - -set_on_failed_callback "FAIL pylint errors" - -if [[ -n "$GITHUB_BASE_REF" ]]; then - echo "Running pylint on changed files..." - git fetch origin "$GITHUB_BASE_REF" - if ! (git diff --name-only --diff-filter=d "origin/$GITHUB_BASE_REF" -z -- '*.py' | xargs -0 --no-run-if-empty pylint --rcfile="$GIT_ROOT/pyproject.toml" --fail-under 9.0); then - FAIL "pylint failed." - exit 1 - fi -fi - -PASS "pylint check passed!" -exit 0 diff --git a/scripts/ci/style/pyright_check.sh b/scripts/ci/style/pyright_check.sh deleted file mode 100755 index 243ce4a103b..00000000000 --- a/scripts/ci/style/pyright_check.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -GIT_ROOT=$(git rev-parse --show-toplevel) - -cd "$GIT_ROOT" || exit - -source ./scripts/ci/helpers.sh - -set_on_failed_callback "FAIL pyright errors" - -if [[ -n "$GITHUB_BASE_REF" ]]; then - echo "Running pyright on changed files..." - git fetch origin "$GITHUB_BASE_REF" - if ! (git diff --name-only --diff-filter=d "origin/$GITHUB_BASE_REF" -z -- '*.py[i]' | xargs -0 --no-run-if-empty pyright); then - FAIL "pyright failed." - exit 1 - fi -fi - -PASS "pyright passed!" -exit 0 diff --git a/scripts/install_hooks.sh b/scripts/install_hooks.sh deleted file mode 100755 index 67dcc54e40a..00000000000 --- a/scripts/install_hooks.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -GIT_ROOT=$(git rev-parse --show-toplevel) - -HOOKS_PATH="$GIT_ROOT/.git/hooks" - -cd "$HOOKS_PATH" || exit - -if [ ! -f "$HOOKS_PATH/commit-msg" ]; then - ln -s "$GIT_ROOT/hooks/commit-msg" . -fi - -if [ ! -f "$HOOKS_PATH/prepare-commit-msg" ]; then - while true; do - read -p "Do you want to setup sign-off commits? Make sure you know what you are doing :) " yn - case $yn in - [Yy]* ) - ln -s "$GIT_ROOT/hooks/prepare-commit-msg" .; break;; - [Nn]* ) exit;; - * ) echo "Please answer yes or no.";; - esac - done -fi - -if [ ! -f "$HOOKS_PATH/pre-commit" ]; then - ln -s "$GIT_ROOT/hooks/pre-commit" . -fi diff --git a/scripts/tools/formatter.sh b/scripts/tools/formatter.sh deleted file mode 100755 index df26887113c..00000000000 --- a/scripts/tools/formatter.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -GIT_ROOT=$(git rev-parse --show-toplevel) - -cd "$GIT_ROOT" || exit 1 - -source ./scripts/ci/helpers.sh - -INFO "(black) Formatting codebase..." - -black --config ./pyproject.toml src tests docs examples - -INFO "(black) Formatting stubs..." - -find src -name "*.pyi" ! -name "*_pb2*" -exec black --pyi --config ./pyproject.toml {} \; - -INFO "(isort) Reordering imports..." - -isort . diff --git a/scripts/tools/linter.sh b/scripts/tools/linter.sh deleted file mode 100755 index 034383eb65c..00000000000 --- a/scripts/tools/linter.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -GIT_ROOT=$(git rev-parse --show-toplevel) - -cd "$GIT_ROOT" || exit 1 - -source ./scripts/ci/helpers.sh - -INFO "(pylint) Linting bentoml..." - -pylint --rcfile="./pyproject.toml" --fail-under 9.5 src - -INFO "(pylint) Linting examples..." - -pylint --rcfile="./pyproject.toml" --fail-under 9.0 --disable=W0621,E0611 examples - -INFO "(pylint) Linting tests..." - -pylint --rcfile="./pyproject.toml" --disable=E0401,F0010 tests diff --git a/scripts/tools/stubs_cleanup.sh b/scripts/tools/stubs_cleanup.sh deleted file mode 100755 index 2e66a2e07eb..00000000000 --- a/scripts/tools/stubs_cleanup.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash - -GIT_ROOT=$(git rev-parse --show-toplevel) -MINIFY_OPTS=( --remove-literal-statements --no-remove-annotations --no-hoist-literals --no-rename-locals --no-remove-object-base --no-convert-posargs-to-args ) - -cd "$GIT_ROOT" || exit 1 - -source ./scripts/ci/helpers.sh - -need_cmd "$EDITOR" || FAIL "You will need an editor to run this script, set your editor with env var: \$EDITOR=" -PROCESSED_TXT="$GIT_ROOT"/typings/processed.txt -: "${EDITOR:=/usr/bin/nano}" - -if [[ ! -f "$PROCESSED_TXT" ]]; then - touch "$PROCESSED_TXT" -fi - -need_cmd black || (echo "black command not found, install dev dependencies with \`make install-dev-deps\`"; exit 1); - -if [[ $(uname) == "Darwin" ]]; then - SED_OPTS=( -E -i '' ) -else - SED_OPTS=( -E -i ) -fi - -INFO "(pyminify) reducing stubs size..." - -for file in $(git ls-files | grep -e "**.pyi$"); do - if [ ! -z $(grep "$file" "$PROCESSED_TXT") ]; then - PASS "$file already minified, skipping..." - continue - else - INFO "Processing $file ..." - INFO "Removing pyright bugs..." - sed "${SED_OPTS[@]}" "s/],:/]/g; s/,,/,/g; s/]\\\n .../]: .../g" "$file" - cp "$file" "$file".bak && rm "$file" - if ! pyminify "${MINIFY_OPTS[@]}" "$file".bak > "$file"; then - FAIL "Unable to processed $file, reverting to previous state. One can also use https://python-minifier.com/ to test where the problem may be. Make sure to match ${MINIFY_OPTS[@]}\nExitting now..." - rm "$file" - mv "$file".bak "$file" - exit 1 - fi - black --fast --config "$GIT_ROOT"/pyproject.toml --pyi "$file" - printf "%s\n" "$file" >> "$PROCESSED_TXT" - \rm "$file".bak - PASS "Finished processing $file..." - fi -done diff --git a/scripts/tools/type_checker.sh b/scripts/tools/type_checker.sh deleted file mode 100755 index 99303175a70..00000000000 --- a/scripts/tools/type_checker.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -GIT_ROOT=$(git rev-parse --show-toplevel) - -cd "$GIT_ROOT" || exit 1 - -source ./scripts/ci/helpers.sh - -INFO "(pyright) Typechecking codebase..." - -pyright -p src/ -w diff --git a/scripts/tools/workflow_cleanup.sh b/scripts/workflow_cleanup.sh similarity index 100% rename from scripts/tools/workflow_cleanup.sh rename to scripts/workflow_cleanup.sh diff --git a/src/bentoml/_internal/frameworks/picklable.py b/src/bentoml/_internal/frameworks/picklable.py index 7924a9c8e8c..a5315257d1d 100644 --- a/src/bentoml/_internal/frameworks/picklable.py +++ b/src/bentoml/_internal/frameworks/picklable.py @@ -12,8 +12,6 @@ from bentoml.models import Model from bentoml.models import ModelContext from bentoml.exceptions import NotFound -from bentoml.exceptions import BentoMLException -from bentoml.exceptions import MissingDependencyException from ..models import PKL_EXT from ..models import SAVE_NAMESPACE @@ -62,7 +60,7 @@ def load_model(bento_model: str | Tag | Model) -> ModelType: bento_model = get(bento_model) if bento_model.info.module not in (MODULE_NAME, __name__): - raise BentoMLException( + raise NotFound( f"Model {bento_model.tag} was saved with module {bento_model.info.module}, not loading with {MODULE_NAME}." ) diff --git a/src/bentoml/_internal/frameworks/sklearn.py b/src/bentoml/_internal/frameworks/sklearn.py index ff79a66d418..12723442907 100644 --- a/src/bentoml/_internal/frameworks/sklearn.py +++ b/src/bentoml/_internal/frameworks/sklearn.py @@ -10,7 +10,6 @@ from bentoml.models import Model from bentoml.models import ModelContext from bentoml.exceptions import NotFound -from bentoml.exceptions import BentoMLException from bentoml.exceptions import MissingDependencyException from ..types import LazyType @@ -81,7 +80,7 @@ def load_model( bento_model = get(bento_model) if bento_model.info.module not in (MODULE_NAME, __name__): - raise BentoMLException( + raise NotFound( f"Model {bento_model.tag} was saved with module {bento_model.info.module}, not loading with {MODULE_NAME}." ) model_file = bento_model.path_of(MODEL_FILENAME) diff --git a/src/bentoml/grpc/buf.yaml b/src/bentoml/grpc/buf.yaml index dcb9aef3df4..d5f80473628 100644 --- a/src/bentoml/grpc/buf.yaml +++ b/src/bentoml/grpc/buf.yaml @@ -10,11 +10,8 @@ lint: - DIRECTORY_SAME_PACKAGE - RPC_REQUEST_STANDARD_NAME - RPC_RESPONSE_STANDARD_NAME - ignore_only: - DEFAULT: - - src/bentoml/grpc/v1alpha1/service_test.proto - ENUM_VALUE_PREFIX: - - src/bentoml/grpc/v1alpha1/service.proto + - ENUM_VALUE_PREFIX + - PACKAGE_DIRECTORY_MATCH enum_zero_value_suffix: _UNSPECIFIED rpc_allow_same_request_response: true rpc_allow_google_protobuf_empty_requests: true diff --git a/tests/e2e/README.md b/tests/e2e/README.md index b2ac7728d49..1fe49d1957e 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -6,26 +6,13 @@ This folder contains end-to-end test suite. To create a new test suite (for simplicity let's call our test suite `qa`), do the following: -1. Navigate to [`config.yml`](../../scripts/ci/config.yml) and add the E2E definition: - -```yaml -qa: - <<: *tmpl - root_test_dir: "tests/e2e/qa" - is_dir: true - type_tests: "e2e" - dependencies: # add required Python dependencies here. - - Pillow - - pydantic - - grpcio-status -``` - -2. Create the folder `qa` with the following project structure: +1. Create the folder `qa` with the following project structure: ```bash . ├── bentofile.yaml ├── train.py +├── requirements.txt ... ├── service.py └── tests @@ -38,7 +25,7 @@ qa: > Note that files under `tests` are merely examples, feel free to add any types of > additional tests. -3. Create a `train.py`: +2. Create a `train.py`: ```python if __name__ == "__main__": @@ -59,7 +46,7 @@ if __name__ == "__main__": ) ``` -4. Inside `tests/conftest.py`, create a `host` fixture like so: +3. Inside `tests/conftest.py`, create a `host` fixture like so: ```python # pylint: disable=unused-argument @@ -107,8 +94,14 @@ def host( yield _host ``` +4. Install your new e2e test's requirements: + +```bash +pip install -r tests/e2e/qa/requirements.txt +``` + 5. To run the tests, navigate to `GIT_ROOT` (root directory of bentoml), and call: ```bash -./scripts/ci/run_tests.sh qa +pytest tests/e2e/qa ``` diff --git a/tests/e2e/bento_server_grpc/requirements.txt b/tests/e2e/bento_server_grpc/requirements.txt new file mode 100644 index 00000000000..89f864f54d7 --- /dev/null +++ b/tests/e2e/bento_server_grpc/requirements.txt @@ -0,0 +1,2 @@ +Pillow +pydantic diff --git a/tests/e2e/bento_server_grpc/tests/test_descriptors.py b/tests/e2e/bento_server_grpc/tests/test_descriptors.py index d6f27c96cf1..bff8da31238 100644 --- a/tests/e2e/bento_server_grpc/tests/test_descriptors.py +++ b/tests/e2e/bento_server_grpc/tests/test_descriptors.py @@ -13,7 +13,7 @@ from bentoml.testing.grpc import create_channel from bentoml.testing.grpc import async_client_call from bentoml.testing.grpc import randomize_pb_ndarray -from bentoml._internal.utils import LazyType +from bentoml._internal.types import LazyType from bentoml._internal.utils import LazyLoader if TYPE_CHECKING: diff --git a/tests/integration/frameworks/conftest.py b/tests/integration/frameworks/conftest.py index f95389fafc5..7480a9da07b 100644 --- a/tests/integration/frameworks/conftest.py +++ b/tests/integration/frameworks/conftest.py @@ -86,7 +86,7 @@ def test_inputs(framework: str | None) -> list[tuple[ModuleType, FrameworkTestMo ) except ModuleNotFoundError as e: logger.warning( - f"Failed to find test module for framework {framework_name} (tests.integration.frameworks.models.{framework_name})" + f"Failed to find test module for framework {framework_name} (tests.integration.frameworks.models.{framework_name}): {e}" ) return [ diff --git a/tests/integration/frameworks/mlflow/MNIST/.gitignore b/tests/integration/frameworks/mlflow/MNIST/.gitignore new file mode 100644 index 00000000000..72206f03165 --- /dev/null +++ b/tests/integration/frameworks/mlflow/MNIST/.gitignore @@ -0,0 +1,4 @@ +dataset +*.ckpt +lightning_logs +mlruns diff --git a/tests/integration/frameworks/mlflow/NestedMNIST/MLproject b/tests/integration/frameworks/mlflow/MNIST/MLproject similarity index 100% rename from tests/integration/frameworks/mlflow/NestedMNIST/MLproject rename to tests/integration/frameworks/mlflow/MNIST/MLproject diff --git a/tests/integration/frameworks/mlflow/MNIST/conda.yaml b/tests/integration/frameworks/mlflow/MNIST/conda.yaml new file mode 100644 index 00000000000..17a80a1db8f --- /dev/null +++ b/tests/integration/frameworks/mlflow/MNIST/conda.yaml @@ -0,0 +1,11 @@ +channels: + - conda-forge +dependencies: + - python=3.8.2 + - pip + - pip: + - mlflow + - torchvision>=0.9.1 + - torch>=1.9.0 + - pytorch-lightning==1.7.6 + - protobuf<4.0.0 diff --git a/tests/integration/frameworks/mlflow/NestedMNIST/mnist_autolog_example.py b/tests/integration/frameworks/mlflow/MNIST/mnist_autolog_example.py similarity index 82% rename from tests/integration/frameworks/mlflow/NestedMNIST/mnist_autolog_example.py rename to tests/integration/frameworks/mlflow/MNIST/mnist_autolog_example.py index d084d24623a..5fc70150989 100644 --- a/tests/integration/frameworks/mlflow/NestedMNIST/mnist_autolog_example.py +++ b/tests/integration/frameworks/mlflow/MNIST/mnist_autolog_example.py @@ -1,5 +1,5 @@ # -# Trains an SimpleMNIST digit recognizer using PyTorch Lightning, +# Trains an MNIST digit recognizer using PyTorch Lightning, # and uses Mlflow to log metrics, params and artifacts # NOTE: This example requires you to first install # pytorch-lightning (using pip install pytorch-lightning) @@ -9,9 +9,11 @@ # pylint: disable=unused-argument # pylint: disable=abstract-method import os +import logging from argparse import ArgumentParser import torch +import mlflow import mlflow.pytorch import pytorch_lightning as pl from torch.nn import functional as F @@ -21,16 +23,20 @@ from torch.utils.data import random_split from pytorch_lightning.callbacks import ModelCheckpoint from pytorch_lightning.callbacks import LearningRateMonitor -from pytorch_lightning.metrics.functional import accuracy from pytorch_lightning.callbacks.early_stopping import EarlyStopping +try: + from torchmetrics.functional import accuracy +except ImportError: + from pytorch_lightning.metrics.functional import accuracy + class MNISTDataModule(pl.LightningDataModule): def __init__(self, **kwargs): """ Initialization of inherited lightning data module """ - super(MNISTDataModule, self).__init__() + super().__init__() self.df_train = None self.df_val = None self.df_test = None @@ -46,8 +52,7 @@ def __init__(self, **kwargs): def setup(self, stage=None): """ - Downloads the data, parses it, and splits the data into training, test, and - validation data + Downloads the data, parse it and split the data into train, test, validation data :param stage: Stage - training or testing """ @@ -69,9 +74,7 @@ def create_data_loader(self, df): :return: Returns the constructed dataloader """ return DataLoader( - df, - batch_size=self.args["batch_size"], - num_workers=self.args["num_workers"], + df, batch_size=self.args["batch_size"], num_workers=self.args["num_workers"] ) def train_dataloader(self): @@ -98,7 +101,7 @@ def __init__(self, **kwargs): """ Initializes the network """ - super(LightningMNISTClassifier, self).__init__() + super().__init__() # mnist images are (1, 28, 28) (channels, width, height) self.optimizer = None @@ -232,7 +235,7 @@ def test_epoch_end(self, outputs): :return: output - average test loss """ avg_test_acc = torch.stack([x["test_acc"] for x in outputs]).mean() - self.log("avg_test_acc", avg_test_acc) + self.log("avg_test_acc", avg_test_acc, sync_dist=True) def configure_optimizers(self): """ @@ -281,14 +284,16 @@ def configure_optimizers(self): parser = pl.Trainer.add_argparse_args(parent_parser=parser) parser = LightningMNISTClassifier.add_model_specific_args(parent_parser=parser) - mlflow.pytorch.autolog() - args = parser.parse_args() dict_args = vars(args) - if "accelerator" in dict_args: - if dict_args["accelerator"] == "None": - dict_args["accelerator"] = None + if "strategy" in dict_args: + if dict_args["strategy"] == "None": + dict_args["strategy"] = None + + if "devices" in dict_args: + if dict_args["devices"] == "None": + dict_args["devices"] = None model = LightningMNISTClassifier(**dict_args) @@ -308,9 +313,26 @@ def configure_optimizers(self): lr_logger = LearningRateMonitor() trainer = pl.Trainer.from_argparse_args( - args, - callbacks=[lr_logger, early_stopping, checkpoint_callback], - checkpoint_callback=True, + args, callbacks=[lr_logger, early_stopping, checkpoint_callback] ) + + # It is safe to use `mlflow.pytorch.autolog` in DDP training, as below condition invokes + # autolog with only rank 0 gpu. + + # For CPU Training + if dict_args["devices"] is None or int(dict_args["devices"]) == 0: + mlflow.pytorch.autolog() + elif int(dict_args["devices"]) >= 1 and trainer.global_rank == 0: + # In case of multi gpu training, the training script is invoked multiple times, + # The following condition is needed to avoid multiple copies of mlflow runs. + # When one or more gpus are used for training, it is enough to save + # the model and its parameters using rank 0 gpu. + mlflow.pytorch.autolog() + else: + # This condition is met only for multi-gpu training when the global rank is non zero. + # Since the parameters are already logged using global rank 0 gpu, it is safe to ignore + # this condition. + logging.info("Active run exists.. ") + trainer.fit(model, dm) - trainer.test() + trainer.test(datamodule=dm, ckpt_path="best") diff --git a/tests/integration/frameworks/mlflow/NestedMNIST/conda.yaml b/tests/integration/frameworks/mlflow/NestedMNIST/conda.yaml deleted file mode 100644 index e557cd40c3f..00000000000 --- a/tests/integration/frameworks/mlflow/NestedMNIST/conda.yaml +++ /dev/null @@ -1,10 +0,0 @@ -channels: -- conda-forge -dependencies: -- python=3.8.2 -- pip -- pip: - - mlflow - - torchvision>=0.9.1 - - torch==1.9.0 - - pytorch-lightning==1.4.0 diff --git a/tests/integration/frameworks/mlflow/NestedMNIST/epoch=2-step=2579.ckpt b/tests/integration/frameworks/mlflow/NestedMNIST/epoch=2-step=2579.ckpt deleted file mode 100644 index 90ae1a186e3..00000000000 Binary files a/tests/integration/frameworks/mlflow/NestedMNIST/epoch=2-step=2579.ckpt and /dev/null differ diff --git a/tests/integration/frameworks/mlflow/NestedMNIST/example/conda.yaml b/tests/integration/frameworks/mlflow/NestedMNIST/example/conda.yaml deleted file mode 100644 index f3bb29e05e1..00000000000 --- a/tests/integration/frameworks/mlflow/NestedMNIST/example/conda.yaml +++ /dev/null @@ -1,10 +0,0 @@ -channels: -- conda-forge -dependencies: -- python=3.8.2 -- pip -- pip: - - mlflow - - torch==1.8.0 - - torchvision==0.9.1 - - pytorch-lightning==1.0.2 diff --git a/tests/integration/frameworks/mlflow/NoPyfunc/MLmodel b/tests/integration/frameworks/mlflow/NoPyfunc/MLmodel new file mode 100644 index 00000000000..c6f46ba32c3 --- /dev/null +++ b/tests/integration/frameworks/mlflow/NoPyfunc/MLmodel @@ -0,0 +1,7 @@ +flavors: + sklearn: + pickled_model: model.pkl + serialization_format: cloudpickle + sklearn_version: 1.1.2 +model_uuid: 985e1569d5b741dc9b43741e1b94e823 +utc_time_created: '2022-10-07 06:26:49.386149' diff --git a/tests/integration/frameworks/mlflow/NoPyfunc/conda.yaml b/tests/integration/frameworks/mlflow/NoPyfunc/conda.yaml new file mode 100644 index 00000000000..89ef14fca40 --- /dev/null +++ b/tests/integration/frameworks/mlflow/NoPyfunc/conda.yaml @@ -0,0 +1,11 @@ +channels: + - conda-forge +dependencies: + - python=3.10 + - pip + - pip: + - mlflow + - cloudpickle + - psutil + - scikit-learn +name: no-pyfunc diff --git a/tests/integration/frameworks/mlflow/NoPyfunc/requirements.txt b/tests/integration/frameworks/mlflow/NoPyfunc/requirements.txt new file mode 100644 index 00000000000..15ea8bb454a --- /dev/null +++ b/tests/integration/frameworks/mlflow/NoPyfunc/requirements.txt @@ -0,0 +1,4 @@ +mlflow +cloudpickle>=2.0.0 +psutil>=5.8.0 +scikit-learn>=1.0.2 diff --git a/tests/integration/frameworks/mlflow/SimpleMNIST/MLproject b/tests/integration/frameworks/mlflow/SimpleMNIST/MLproject deleted file mode 100644 index 3a4a7f31477..00000000000 --- a/tests/integration/frameworks/mlflow/SimpleMNIST/MLproject +++ /dev/null @@ -1,30 +0,0 @@ -name: mnist-autolog-example - -conda_env: conda.yaml - -entry_points: - main: - parameters: - max_epochs: {type: int, default: 5} - gpus: {type: int, default: 0} - accelerator: {type str, default: "None"} - batch_size: {type: int, default: 64} - num_workers: {type: int, default: 3} - learning_rate: {type: float, default: 0.001} - patience: {type int, default: 3} - mode: {type str, default: 'min'} - verbose: {type bool, default: True} - monitor: {type str, default: 'val_loss'} - - command: | - python mnist_autolog_example.py \ - --max_epochs {max_epochs} \ - --gpus {gpus} \ - --accelerator {accelerator} \ - --batch_size {batch_size} \ - --num_workers {num_workers} \ - --lr {learning_rate} \ - --es_patience {patience} \ - --es_mode {mode} \ - --es_verbose {verbose} \ - --es_monitor {monitor} diff --git a/tests/integration/frameworks/mlflow/SimpleMNIST/conda.yaml b/tests/integration/frameworks/mlflow/SimpleMNIST/conda.yaml deleted file mode 100644 index e557cd40c3f..00000000000 --- a/tests/integration/frameworks/mlflow/SimpleMNIST/conda.yaml +++ /dev/null @@ -1,10 +0,0 @@ -channels: -- conda-forge -dependencies: -- python=3.8.2 -- pip -- pip: - - mlflow - - torchvision>=0.9.1 - - torch==1.9.0 - - pytorch-lightning==1.4.0 diff --git a/tests/integration/frameworks/mlflow/SimpleMNIST/mnist_autolog_example.py b/tests/integration/frameworks/mlflow/SimpleMNIST/mnist_autolog_example.py deleted file mode 100644 index 6d6279e0087..00000000000 --- a/tests/integration/frameworks/mlflow/SimpleMNIST/mnist_autolog_example.py +++ /dev/null @@ -1,316 +0,0 @@ -# -# Trains an SimpleMNIST digit recognizer using PyTorch Lightning, -# and uses Mlflow to log metrics, params and artifacts -# NOTE: This example requires you to first install -# pytorch-lightning (using pip install pytorch-lightning) -# and mlflow (using pip install mlflow). -# -# pylint: disable=arguments-differ -# pylint: disable=unused-argument -# pylint: disable=abstract-method -import os -from argparse import ArgumentParser - -import torch -import mlflow.pytorch -import pytorch_lightning as pl -from torch.nn import functional as F -from torchvision import datasets -from torchvision import transforms -from torch.utils.data import DataLoader -from torch.utils.data import random_split -from pytorch_lightning.callbacks import ModelCheckpoint -from pytorch_lightning.callbacks import LearningRateMonitor -from pytorch_lightning.metrics.functional import accuracy -from pytorch_lightning.callbacks.early_stopping import EarlyStopping - - -class MNISTDataModule(pl.LightningDataModule): - def __init__(self, **kwargs): - """ - Initialization of inherited lightning data module - """ - super(MNISTDataModule, self).__init__() - self.df_train = None - self.df_val = None - self.df_test = None - self.train_data_loader = None - self.val_data_loader = None - self.test_data_loader = None - self.args = kwargs - - # transforms for images - self.transform = transforms.Compose( - [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))] - ) - - def setup(self, stage=None): - """ - Downloads the data, parse it and split - the data into train, test, validation data - - :param stage: Stage - training or testing - """ - - self.df_train = datasets.MNIST( - "dataset", download=True, train=True, transform=self.transform - ) - self.df_train, self.df_val = random_split(self.df_train, [55000, 5000]) - self.df_test = datasets.MNIST( - "dataset", download=True, train=False, transform=self.transform - ) - - def create_data_loader(self, df): - """ - Generic data loader function - - :param df: Input tensor - - :return: Returns the constructed dataloader - """ - return DataLoader( - df, - batch_size=self.args["batch_size"], - num_workers=self.args["num_workers"], - ) - - def train_dataloader(self): - """ - :return: output - Train data loader for the given input - """ - return self.create_data_loader(self.df_train) - - def val_dataloader(self): - """ - :return: output - Validation data loader for the given input - """ - return self.create_data_loader(self.df_val) - - def test_dataloader(self): - """ - :return: output - Test data loader for the given input - """ - return self.create_data_loader(self.df_test) - - -class LightningMNISTClassifier(pl.LightningModule): - def __init__(self, **kwargs): - """ - Initializes the network - """ - super(LightningMNISTClassifier, self).__init__() - - # mnist images are (1, 28, 28) (channels, width, height) - self.optimizer = None - self.scheduler = None - self.layer_1 = torch.nn.Linear(28 * 28, 128) - self.layer_2 = torch.nn.Linear(128, 256) - self.layer_3 = torch.nn.Linear(256, 10) - self.args = kwargs - - @staticmethod - def add_model_specific_args(parent_parser): - parser = ArgumentParser(parents=[parent_parser], add_help=False) - parser.add_argument( - "--batch_size", - type=int, - default=64, - metavar="N", - help="input batch size for training (default: 64)", - ) - parser.add_argument( - "--num_workers", - type=int, - default=3, - metavar="N", - help="number of workers (default: 3)", - ) - parser.add_argument( - "--lr", - type=float, - default=0.001, - metavar="LR", - help="learning rate (default: 0.001)", - ) - return parser - - def forward(self, x): - """ - :param x: Input data - - :return: output - mnist digit label for the input image - """ - batch_size = x.size()[0] - - # (b, 1, 28, 28) -> (b, 1*28*28) - x = x.view(batch_size, -1) - - # layer 1 (b, 1*28*28) -> (b, 128) - x = self.layer_1(x) - x = torch.relu(x) - - # layer 2 (b, 128) -> (b, 256) - x = self.layer_2(x) - x = torch.relu(x) - - # layer 3 (b, 256) -> (b, 10) - x = self.layer_3(x) - - # probability distribution over labels - x = torch.log_softmax(x, dim=1) - - return x - - def cross_entropy_loss(self, logits, labels): - """ - Initializes the loss function - - :return: output - Initialized cross entropy loss function - """ - return F.nll_loss(logits, labels) - - def training_step(self, train_batch, batch_idx): - """ - Training the data as batches and returns training loss on each batch - - :param train_batch: Batch data - :param batch_idx: Batch indices - - :return: output - Training loss - """ - x, y = train_batch - logits = self.forward(x) - loss = self.cross_entropy_loss(logits, y) - return {"loss": loss} - - def validation_step(self, val_batch, batch_idx): - """ - Performs validation of data in batches - - :param val_batch: Batch data - :param batch_idx: Batch indices - - :return: output - valid step loss - """ - x, y = val_batch - logits = self.forward(x) - loss = self.cross_entropy_loss(logits, y) - return {"val_step_loss": loss} - - def validation_epoch_end(self, outputs): - """ - Computes average validation accuracy - - :param outputs: outputs after every epoch end - - :return: output - average valid loss - """ - avg_loss = torch.stack([x["val_step_loss"] for x in outputs]).mean() - self.log("val_loss", avg_loss, sync_dist=True) - - def test_step(self, test_batch, batch_idx): - """ - Performs test and computes the accuracy of the model - - :param test_batch: Batch data - :param batch_idx: Batch indices - - :return: output - Testing accuracy - """ - x, y = test_batch - output = self.forward(x) - _, y_hat = torch.max(output, dim=1) - test_acc = accuracy(y_hat.cpu(), y.cpu()) - return {"test_acc": test_acc} - - def test_epoch_end(self, outputs): - """ - Computes average test accuracy score - - :param outputs: outputs after every epoch end - - :return: output - average test loss - """ - avg_test_acc = torch.stack([x["test_acc"] for x in outputs]).mean() - self.log("avg_test_acc", avg_test_acc) - - def configure_optimizers(self): - """ - Initializes the optimizer and learning rate scheduler - - :return: output - Initialized optimizer and scheduler - """ - self.optimizer = torch.optim.Adam(self.parameters(), lr=self.args["lr"]) - self.scheduler = { - "scheduler": torch.optim.lr_scheduler.ReduceLROnPlateau( - self.optimizer, - mode="min", - factor=0.2, - patience=2, - min_lr=1e-6, - verbose=True, - ), - "monitor": "val_loss", - } - return [self.optimizer], [self.scheduler] - - -if __name__ == "__main__": - parser = ArgumentParser(description="PyTorch Autolog Mnist Example") - - # Early stopping parameters - parser.add_argument( - "--es_monitor", - type=str, - default="val_loss", - help="Early stopping monitor parameter", - ) - - parser.add_argument( - "--es_mode", type=str, default="min", help="Early stopping mode parameter" - ) - - parser.add_argument( - "--es_verbose", type=bool, default=True, help="Early stopping verbose parameter" - ) - - parser.add_argument( - "--es_patience", type=int, default=3, help="Early stopping patience parameter" - ) - - parser = pl.Trainer.add_argparse_args(parent_parser=parser) - parser = LightningMNISTClassifier.add_model_specific_args(parent_parser=parser) - - mlflow.pytorch.autolog() - - args = parser.parse_args() - dict_args = vars(args) - - if "accelerator" in dict_args: - if dict_args["accelerator"] == "None": - dict_args["accelerator"] = None - - model = LightningMNISTClassifier(**dict_args) - - dm = MNISTDataModule(**dict_args) - dm.setup(stage="fit") - - early_stopping = EarlyStopping( - monitor=dict_args["es_monitor"], - mode=dict_args["es_mode"], - verbose=dict_args["es_verbose"], - patience=dict_args["es_patience"], - ) - - checkpoint_callback = ModelCheckpoint( - dirpath=os.getcwd(), save_top_k=1, verbose=True, monitor="val_loss", mode="min" - ) - lr_logger = LearningRateMonitor() - - trainer = pl.Trainer.from_argparse_args( - args, - callbacks=[lr_logger, early_stopping, checkpoint_callback], - checkpoint_callback=True, - ) - trainer.fit(model, dm) - trainer.test() diff --git a/tests/integration/frameworks/mlflow/sklearn_clf/MLmodel b/tests/integration/frameworks/mlflow/sklearn_clf/MLmodel deleted file mode 100644 index f2b058d45e5..00000000000 --- a/tests/integration/frameworks/mlflow/sklearn_clf/MLmodel +++ /dev/null @@ -1,12 +0,0 @@ -flavors: - python_function: - env: conda.yaml - loader_module: mlflow.sklearn - model_path: model.pkl - python_version: 3.9.4 - sklearn: - pickled_model: model.pkl - serialization_format: cloudpickle - sklearn_version: 1.0.2 -model_uuid: 985e1569d5b741dc9b43741e1b94e823 -utc_time_created: '2022-01-25 06:26:49.386149' diff --git a/tests/integration/frameworks/mlflow/sklearn_clf/conda.yaml b/tests/integration/frameworks/mlflow/sklearn_clf/conda.yaml deleted file mode 100644 index 8f357886a29..00000000000 --- a/tests/integration/frameworks/mlflow/sklearn_clf/conda.yaml +++ /dev/null @@ -1,11 +0,0 @@ -channels: -- conda-forge -dependencies: -- python=3.9.4 -- pip -- pip: - - mlflow - - cloudpickle==2.0.0 - - psutil==5.8.0 - - scikit-learn==1.0.2 -name: mlflow-env diff --git a/tests/integration/frameworks/mlflow/sklearn_clf/model.pkl b/tests/integration/frameworks/mlflow/sklearn_clf/model.pkl deleted file mode 100644 index 0ee4b878ee3..00000000000 Binary files a/tests/integration/frameworks/mlflow/sklearn_clf/model.pkl and /dev/null differ diff --git a/tests/integration/frameworks/mlflow/sklearn_clf/requirements.txt b/tests/integration/frameworks/mlflow/sklearn_clf/requirements.txt deleted file mode 100644 index 40626ef023a..00000000000 --- a/tests/integration/frameworks/mlflow/sklearn_clf/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -mlflow -cloudpickle==2.0.0 -psutil==5.8.0 -scikit-learn==1.0.2 \ No newline at end of file diff --git a/tests/integration/frameworks/mlflow/test_apis.py b/tests/integration/frameworks/mlflow/test_apis.py new file mode 100644 index 00000000000..0e783610c12 --- /dev/null +++ b/tests/integration/frameworks/mlflow/test_apis.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +import os +import typing as t +from typing import TYPE_CHECKING +from pathlib import Path + +import numpy as np +import mlflow +import pytest +import mlflow.models +import mlflow.sklearn +import mlflow.tracking +from sklearn.datasets import load_iris +from sklearn.neighbors import KNeighborsClassifier + +import bentoml +from bentoml.exceptions import NotFound +from bentoml.exceptions import BentoMLException +from bentoml._internal.models.model import ModelContext + +if TYPE_CHECKING: + from sklearn.utils import Bunch + + from bentoml import Tag + from bentoml._internal import external_typing as ext + +MODEL_NAME = __name__.split(".")[-1] + +# fmt: off +res = np.array( + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2] +) +# fmt: on + +iris: Bunch = t.cast("Bunch", load_iris()) +X: ext.NpNDArray = iris.data[:, :4] +Y: ext.NpNDArray = iris.target + + +@pytest.fixture(name="URI") +def iris_clf_model(tmp_path: Path) -> Path: + URI = tmp_path / "IrisClf" + model = KNeighborsClassifier() + model.fit(X, Y) + mlflow.sklearn.save_model(model, URI.resolve()) + return URI + + +# MLFlow db initialization spews SQLAlchemy deprecation warnings +@pytest.mark.filterwarnings("ignore:.*:sqlalchemy.exc.SADeprecationWarning") +def test_mlflow_save_load(URI: Path, tmp_path: Path): + tracking_db = tmp_path / "mlruns.db" + mlflow.set_tracking_uri(f"sqlite:///{tracking_db}") + client = mlflow.tracking.MlflowClient() + mv = mlflow.register_model(str(URI), "IrisClf") + client.transition_model_version_stage( + name="IrisClf", + version=mv.version, + stage="Staging", + ) + bento_model = bentoml.mlflow.import_model(MODEL_NAME, str(URI.resolve())) + # make sure the model can be imported with models:/ + model_uri = bentoml.mlflow.import_model(MODEL_NAME, "models:/IrisClf/Staging") + + pyfunc = bentoml.mlflow.load_model(bento_model.tag) + np.testing.assert_array_equal(pyfunc.predict(X), res) + np.testing.assert_array_equal( + bentoml.mlflow.load_model(model_uri.tag).predict(X), res + ) + + +def test_wrong_module_load(): + with bentoml.models.create( + "wrong_module", + module=__name__, + context=ModelContext("wrong_module", {"wrong_module": "1.0.0"}), + signatures={}, + ) as ctx: + tag = ctx.tag + model = ctx + + with pytest.raises( + NotFound, match=f"Model {tag} was saved with module {__name__}, " + ): + bentoml.mlflow.get(tag) + + with pytest.raises( + NotFound, match=f"Model {tag} was saved with module {__name__}, " + ): + bentoml.mlflow.load_model(tag) + + with pytest.raises( + NotFound, match=f"Model {tag} was saved with module {__name__}, " + ): + bentoml.mlflow.load_model(model) + + +def test_invalid_import(): + uri = Path(__file__).parent / "NoPyfunc" + with pytest.raises( + BentoMLException, + match="does not support the required python_function flavor", + ): + _ = bentoml.mlflow.import_model("NoPyfunc", str(uri.resolve())) + + +@pytest.fixture(name="no_mlmodel") +def fixture_no_mlmodel(URI: Path) -> Tag: + bento_model = bentoml.mlflow.import_model("IrisClf", str(URI)) + info = bentoml.models.get(bento_model.tag) + os.remove(str(Path(info.path, "mlflow_model", "MLmodel").resolve())) + return bento_model.tag + + +def test_invalid_load(no_mlmodel: Tag): + with pytest.raises(FileNotFoundError): + _ = bentoml.mlflow.load_model(no_mlmodel) + + +def test_invalid_signatures_model(URI: Path): + with pytest.raises( + BentoMLException, + match=f"MLflow pyfunc model support only the `predict` method, *", + ): + _ = bentoml.mlflow.import_model( + MODEL_NAME, + str(URI), + signatures={ + "asdf": {"batchable": True}, + "no_predict": {"batchable": False}, + }, + ) + + +def test_mlflow_load_runner(URI: Path): + bento_model = bentoml.mlflow.import_model(MODEL_NAME, str(URI)) + runner = bentoml.mlflow.get(bento_model.tag).to_runner() + runner.init_local() + + assert bento_model.tag == runner.models[0].tag + + np.testing.assert_array_equal(runner.run(X), res) + + +def test_mlflow_invalid_import_mlproject(): + uri = Path(__file__).parent / "MNIST" + with pytest.raises(BentoMLException): + _ = bentoml.mlflow.import_model(MODEL_NAME, str(uri)) + + +def test_get_mlflow_model(URI: Path): + bento_model = bentoml.mlflow.import_model(MODEL_NAME, str(URI)) + mlflow_model = bentoml.mlflow.get_mlflow_model(bento_model.tag) + assert isinstance(mlflow_model, mlflow.models.Model) diff --git a/tests/integration/frameworks/mlflow/test_mlflow_save_load.py b/tests/integration/frameworks/mlflow/test_mlflow_save_load.py deleted file mode 100644 index 675d03b0d82..00000000000 --- a/tests/integration/frameworks/mlflow/test_mlflow_save_load.py +++ /dev/null @@ -1,96 +0,0 @@ -import os -from pathlib import Path - -import numpy as np -import psutil -import pytest -import mlflow.sklearn - -import bentoml -from bentoml.exceptions import BentoMLException -from tests.utils.frameworks.sklearn_utils import sklearn_model_data - -current_file = Path(__file__).parent - -MODEL_NAME = __name__.split(".")[-1] - -# fmt: off -res_arr = np.array( - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2] -) -# fmt: on - -# MLFlow db initialization spews SQLAlchemy deprecation warnings -@pytest.mark.filterwarnings("ignore:.*:sqlalchemy.exc.SADeprecationWarning") -def test_mlflow_save_load(): - (model, data) = sklearn_model_data() - uri = Path(current_file, "sklearn_clf") - tracking_db = Path(current_file, "mlruns.db") - if not uri.exists(): - mlflow.sklearn.save_model(model, uri.resolve()) - mlflow.set_tracking_uri("sqlite:///" + str(tracking_db)) - client = mlflow.tracking.MlflowClient() - v = mlflow.register_model(str(uri), "sklearn_clf") - client.transition_model_version_stage( - name="sklearn_clf", version=v.version, stage="Staging" - ) - - bento_model = bentoml.mlflow.import_model(MODEL_NAME, str(uri.resolve())) - model_info = bentoml.models.get(bento_model.tag) - - loaded = bentoml.mlflow.load_model(model_info.tag) - np.testing.assert_array_equal(loaded.predict(data), res_arr) # noqa - - -@pytest.fixture() -def invalid_save_with_no_mlmodel(): - uri = Path(current_file, "sklearn_clf").resolve() - bento_model = bentoml.mlflow.import_model("sklearn_clf", str(uri)) - info = bentoml.models.get(bento_model.tag) - os.remove(str(Path(info.path, "mlflow_model", "MLmodel").resolve())) - return bento_model.tag - - -def test_invalid_load(invalid_save_with_no_mlmodel): - with pytest.raises(FileNotFoundError): - _ = bentoml.mlflow.load_model(invalid_save_with_no_mlmodel) - - -def test_mlflow_load_runner(): - (_, data) = sklearn_model_data() - uri = Path(current_file, "sklearn_clf").resolve() - bento_model = bentoml.mlflow.import_model(MODEL_NAME, str(uri)) - runner = bentoml.mlflow.get(bento_model.tag).to_runner() - runner.init_local() - - assert bento_model.tag == runner.models[0].tag - - res = runner.predict.run(data) - assert all(res == res_arr) - - -@pytest.mark.parametrize( - "uri", - [ - Path(current_file, "SimpleMNIST").resolve(), - Path(current_file, "NestedMNIST").resolve(), - ], -) -def test_mlflow_invalid_import_mlproject(uri): - with pytest.raises(BentoMLException): - _ = bentoml.mlflow.import_model(MODEL_NAME, str(uri)) - - -def test_mlflow_import_models_url(): - tracking_db = Path(current_file, "mlruns.db") - mlflow.set_tracking_uri("sqlite:///" + str(tracking_db)) - _ = bentoml.mlflow.import_model(MODEL_NAME, "models:/sklearn_clf/Staging") diff --git a/tests/integration/frameworks/models/fastai.py b/tests/integration/frameworks/models/fastai.py index 1a3d2dd52f7..93d8c18c4be 100644 --- a/tests/integration/frameworks/models/fastai.py +++ b/tests/integration/frameworks/models/fastai.py @@ -1,37 +1,102 @@ from __future__ import annotations -from typing import Any -from typing import Callable +import typing as t from typing import TYPE_CHECKING import numpy as np +import pandas as pd +import torch.nn as nn +from fastai.learner import Learner +from fastai.metrics import accuracy +from sklearn.datasets import load_iris +from fastai.data.block import DataBlock +from fastai.torch_core import Module from fastai.torch_core import set_seed +from fastai.tabular.all import tabular_learner +from fastai.tabular.all import TabularDataLoaders import bentoml -from tests.utils.frameworks.fastai_utils import X -from tests.utils.frameworks.fastai_utils import SEED -from tests.utils.frameworks.fastai_utils import custom_model -from tests.utils.frameworks.fastai_utils import tabular_model from . import FrameworkTestModel from . import FrameworkTestModelInput as Input from . import FrameworkTestModelConfiguration as Config if TYPE_CHECKING: + from sklearn.utils import Bunch import bentoml._internal.external_typing as ext framework = bentoml.fastai +SEED = 123 + set_seed(SEED, reproducible=True) +iris: Bunch = t.cast("Bunch", load_iris()) +X = pd.DataFrame( + t.cast("ext.NpNDArray", iris.data[:, :2]), + columns=t.cast("list[str]", iris.feature_names[:2]), +) +y = pd.Series(t.cast("ext.NpNDArray", iris.target), name="label") + + +class LinearModel(nn.Module): + def __init__(self): + super().__init__() + self.linear = nn.Linear(5, 1, bias=False) + nn.init.ones_(self.linear.weight) + + def forward(self, x: t.Any): + return self.linear(x) + + +class Loss(Module): + reduction = "none" + + def forward(self, x: t.Any, _y: t.Any): + return x + + def activation(self, x: t.Any): + return x + + def decodes(self, x: t.Any): + return x + + +def tabular_model() -> Learner: + dl = TabularDataLoaders.from_df( + df=pd.concat([X, y], axis=1), + cont_names=list(X.columns), + y_names="label", + num_workers=0, + ) + model = tabular_learner(dl, metrics=accuracy, layers=[3]) + model.fit(1) + return model + + +def custom_model(): + def get_items(_: t.Any) -> ext.NpNDArray: + return np.ones([5, 5], np.float32) + + model = LinearModel() + loss = Loss() + + dblock = DataBlock(get_items=get_items, get_y=np.sum) + dls = dblock.datasets(None).dataloaders() + learner = Learner(dls, model, loss) + learner.fit(1) + return learner + def inputs(x: list[ext.NpNDArray]) -> list[ext.NpNDArray]: return list(map(lambda y: y.astype(np.float32), x)) -def close_to(expected: float) -> Callable[[tuple[Any, Any, ext.NpNDArray]], np.bool_]: - def check(out: tuple[Any, Any, ext.NpNDArray]) -> np.bool_: +def close_to( + expected: float, +) -> t.Callable[[tuple[t.Any, t.Any, ext.NpNDArray]], np.bool_]: + def check(out: tuple[t.Any, t.Any, ext.NpNDArray]) -> np.bool_: return np.isclose(out[-1].squeeze().item(), expected).all() return check diff --git a/tests/integration/frameworks/models/keras_tf2.py b/tests/integration/frameworks/models/keras.py similarity index 100% rename from tests/integration/frameworks/models/keras_tf2.py rename to tests/integration/frameworks/models/keras.py diff --git a/tests/integration/frameworks/models/picklable_model.py b/tests/integration/frameworks/models/picklable_model.py new file mode 100644 index 00000000000..ecbbb3b0d79 --- /dev/null +++ b/tests/integration/frameworks/models/picklable_model.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import numpy as np + +import bentoml + +from . import FrameworkTestModel +from . import FrameworkTestModelInput as Input +from . import FrameworkTestModelConfiguration as Config + +framework = bentoml.picklable_model + + +class PredictModel: + def predict(self, some_integer: int): + return some_integer**2 + + def batch_predict(self, some_integer: list[int]): + return list(map(lambda x: x**2, some_integer)) + + +def fn(x: int) -> int: + return x + 1 + + +pickle_model = FrameworkTestModel( + name="pickable_model", + save_kwargs={ + "signatures": { + "predict": {"batchable": False}, + "batch_predict": {"batchable": True}, + }, + "metadata": {"model": "PredictModel", "test": True}, + "custom_objects": {"func": fn}, + }, + model=PredictModel(), + configurations=[ + Config( + test_inputs={ + "predict": [ + Input(input_args=[4], expected=np.array([16])), + ], + "batch_predict": [ + Input(input_args=[[3, 9]], expected=[9, 81]), + ], + }, + ), + ], +) + +models: list[FrameworkTestModel] = [pickle_model] diff --git a/tests/integration/frameworks/models/pytorch.py b/tests/integration/frameworks/models/pytorch.py index 6674e399b4c..4e29cfba8c0 100644 --- a/tests/integration/frameworks/models/pytorch.py +++ b/tests/integration/frameworks/models/pytorch.py @@ -1,11 +1,12 @@ from __future__ import annotations +import typing as t + import numpy as np import torch -import torch.nn +import torch.nn as nn import bentoml -from tests.utils.frameworks.pytorch_utils import LinearModel from . import FrameworkTestModel from . import FrameworkTestModelInput as Input @@ -19,11 +20,19 @@ expected_output = 5 -model = LinearModel() +class LinearModel(nn.Module): + def __init__(self): + super().__init__() + self.linear = nn.Linear(5, 1, bias=False) + nn.init.ones_(self.linear.weight) + + def forward(self, x: t.Any): + return self.linear(x) + pytorch_model = FrameworkTestModel( name="pytorch", - model=model, + model=LinearModel(), configurations=[ Config( test_inputs={ diff --git a/tests/integration/frameworks/models/sklearn.py b/tests/integration/frameworks/models/sklearn.py new file mode 100644 index 00000000000..a7e4a99a1bd --- /dev/null +++ b/tests/integration/frameworks/models/sklearn.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import typing as t +from typing import TYPE_CHECKING + +import numpy as np +from sklearn.datasets import load_iris +from sklearn.ensemble import RandomForestClassifier + +import bentoml + +from . import FrameworkTestModel +from . import FrameworkTestModelInput as Input +from . import FrameworkTestModelConfiguration as Config + +if TYPE_CHECKING: + from sklearn.utils import Bunch + + from bentoml._internal import external_typing as ext + +iris = t.cast("Bunch", load_iris()) +X: ext.NpNDArray = iris.data[:, :4] +y: ext.NpNDArray = iris.target + +framework = bentoml.sklearn + +# fmt: off +res = np.array( + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2] +) +# fmt: on + +random_forest_classifier = FrameworkTestModel( + name="classification", + save_kwargs={ + "signatures": { + "predict": {"batchable": False}, + } + }, + model=RandomForestClassifier().fit(X, y), + configurations=[ + Config( + test_inputs={ + "predict": [ + Input( + input_args=[X], + expected=res, + ), + ], + }, + ), + ], +) + +models: list[FrameworkTestModel] = [random_forest_classifier] diff --git a/tests/integration/frameworks/models/tensorflow_v2.py b/tests/integration/frameworks/models/tensorflow.py similarity index 98% rename from tests/integration/frameworks/models/tensorflow_v2.py rename to tests/integration/frameworks/models/tensorflow.py index 4ca6055e7d1..a8bd12752ef 100644 --- a/tests/integration/frameworks/models/tensorflow_v2.py +++ b/tests/integration/frameworks/models/tensorflow.py @@ -159,7 +159,7 @@ def make_keras_functional_model() -> tf.keras.Model: FrameworkTestModel( name="tf2", model=model, - # save_kwargs={"signature": {"__call__": {"batchable": True, "batchdim": 0}}}, + save_kwargs={"signatures": {"__call__": {"batchable": True, "batch_dim": 0}}}, configurations=[ Config( test_inputs={ diff --git a/tests/integration/frameworks/test_fastai_unit.py b/tests/integration/frameworks/test_fastai_unit.py index 475763cb55f..28053a03158 100644 --- a/tests/integration/frameworks/test_fastai_unit.py +++ b/tests/integration/frameworks/test_fastai_unit.py @@ -1,6 +1,7 @@ from __future__ import annotations import re +import typing as t import logging from typing import TYPE_CHECKING from unittest.mock import Mock @@ -9,6 +10,7 @@ import torch import pytest +import torch.nn as nn import torch.functional as F from fastai.data.core import TfmdDL from fastai.data.core import Datasets @@ -21,8 +23,7 @@ import bentoml from bentoml.exceptions import InvalidArgument from bentoml.exceptions import BentoMLException -from tests.utils.frameworks.fastai_utils import custom_model -from tests.utils.frameworks.pytorch_utils import LinearModel +from tests.integration.frameworks.models.fastai import custom_model if TYPE_CHECKING: from unittest.mock import MagicMock @@ -32,24 +33,29 @@ learner = custom_model() +class LinearModel(nn.Module): + def forward(self, x: t.Any) -> t.Any: + return x + + class _FakeLossFunc(Module): reduction = "none" - def forward(self, x, y): + def forward(self, x: t.Any, y: t.Any): return F.mse_loss(x, y) - def activation(self, x): + def activation(self, x: t.Any): return x + 1 - def decodes(self, x): + def decodes(self, x: t.Any): return 2 * x class _Add1(Transform): - def encodes(self, x): + def encodes(self, x: t.Any): return x + 1 - def decodes(self, x): + def decodes(self, x: t.Any): return x - 1 @@ -60,9 +66,9 @@ def decodes(self, x): def test_raise_exceptions(): - with pytest.raises(BentoMLException) as excinfo: + with pytest.raises(BentoMLException) as exc: bentoml.fastai.save_model("invalid_learner", LinearModel()) # type: ignore (testing exception) - assert "does not support saving pytorch" in str(excinfo.value) + assert "does not support saving pytorch" in str(exc.value) class ForbiddenType: pass diff --git a/tests/integration/frameworks/test_picklable_model_impl.py b/tests/integration/frameworks/test_picklable_model_impl.py deleted file mode 100644 index c4e89d7320a..00000000000 --- a/tests/integration/frameworks/test_picklable_model_impl.py +++ /dev/null @@ -1,88 +0,0 @@ -import typing as t -from typing import TYPE_CHECKING - -import numpy as np -import pytest - -import bentoml -import bentoml.models - -if TYPE_CHECKING: - from bentoml._internal.store import Tag - - -class MyCoolModel: - def predict(self, some_integer: int): - return some_integer**2 - - def batch_predict(self, some_integer: t.List[int]): - return list(map(lambda x: x**2, some_integer)) - - -def save_test_model( - metadata: t.Dict[str, t.Any], - labels: t.Optional[t.Dict[str, str]] = None, - custom_objects: t.Optional[t.Dict[str, t.Any]] = None, - external_modules: t.Optional[t.List[t.Any]] = None, -) -> "Tag": - model_to_save = MyCoolModel() - bento_model = bentoml.picklable_model.save_model( - "test_picklable_model", - model_to_save, - signatures={ - "predict": {"batchable": False}, - "batch_predict": {"batchable": True}, - }, - metadata=metadata, - labels=labels, - custom_objects=custom_objects, - external_modules=external_modules, - ) - return bento_model - - -@pytest.mark.parametrize( - "metadata", - [({"model": "PicklableModel", "test": True})], -) -def test_picklable_model_save_load( - metadata: t.Dict[str, t.Any], -) -> None: - - labels = {"stage": "dev"} - - def custom_f(x: int) -> int: - return x + 1 - - bentomodel = save_test_model( - metadata, labels=labels, custom_objects={"func": custom_f} - ) - assert bentomodel.info.metadata is not None - for k in labels.keys(): - assert labels[k] == bentomodel.info.labels[k] - assert bentomodel.custom_objects["func"](3) == custom_f(3) - - loaded_model = bentoml.picklable_model.load_model(bentomodel.tag) - assert isinstance(loaded_model, MyCoolModel) - assert loaded_model.predict(4) == np.array([16]) - - -def test_picklable_runner() -> None: - bento_model = save_test_model({}) - runner = bento_model.to_runner() - runner.init_local() - - assert runner.models[0].tag == bento_model.tag - assert runner.predict.run(3) == np.array([9]) - assert runner.batch_predict.run([3, 9]) == [9, 81] - - -def test_picklable_model_default_signature() -> None: - bento_model = bentoml.picklable_model.save_model( - "test_pickle_model", lambda x: x**2, metadata={} - ) - runner = bento_model.to_runner() - runner.init_local() - - assert runner.models[0].tag == bento_model.tag - assert runner.run(3) == np.array([9]) diff --git a/tests/integration/frameworks/test_sklearn_impl.py b/tests/integration/frameworks/test_sklearn_impl.py deleted file mode 100644 index 2ffd5708b37..00000000000 --- a/tests/integration/frameworks/test_sklearn_impl.py +++ /dev/null @@ -1,91 +0,0 @@ -from __future__ import annotations - -import typing as t -from types import ModuleType -from typing import TYPE_CHECKING -from pathlib import Path - -import numpy as np -import pytest -from sklearn.ensemble import RandomForestClassifier - -import bentoml -import bentoml.models -from tests.utils.frameworks.sklearn_utils import sklearn_model_data - -# fmt: off -res_arr = np.array( - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2] -) - -# fmt: on -if TYPE_CHECKING: - from bentoml import Tag - - -def save_test_model( - metadata: t.Dict[str, t.Any], - labels: t.Optional[t.Dict[str, str]] = None, - custom_objects: t.Optional[t.Dict[str, t.Any]] = None, - external_modules: t.List[ModuleType] | None = None, -) -> "Tag": - model, _ = sklearn_model_data(clf=RandomForestClassifier) - tag_info = bentoml.sklearn.save_model( - "test_sklearn_model", - model, - metadata=metadata, - labels=labels, - custom_objects=custom_objects, - external_modules=external_modules, - ) - return tag_info - - -@pytest.mark.parametrize( - "metadata", - [ - ({"model": "Sklearn", "test": True}), - ({"acc": 0.876}), - ], -) -def test_sklearn_save_load(metadata: t.Dict[str, t.Any]) -> None: - - labels = {"stage": "dev"} - - def custom_f(x: int) -> int: - return x + 1 - - _, data = sklearn_model_data(clf=RandomForestClassifier) - bentomodel = save_test_model( - metadata, labels=labels, custom_objects={"func": custom_f} - ) - assert bentomodel.info.metadata is not None - assert any(f.suffix == ".pkl" for f in Path(bentomodel.path).iterdir()) - for k in labels.keys(): - assert labels[k] == bentomodel.info.labels[k] - assert bentomodel.custom_objects["func"](3) == custom_f(3) - - loaded = bentoml.sklearn.load_model(bentomodel.tag) - - assert isinstance(loaded, RandomForestClassifier) - - np.testing.assert_array_equal(loaded.predict(data), res_arr) - - -def test_sklearn_runner() -> None: - _, data = sklearn_model_data(clf=RandomForestClassifier) - bento_model = save_test_model({}) - runner = bento_model.to_runner() - runner.init_local() - - assert runner.models[0].tag == bento_model.tag - assert runner.scheduled_worker_count == 1 - - res = runner.run(data) - assert (res == res_arr).all() diff --git a/tests/integration/frameworks/test_tensorflow_v2_unit.py b/tests/integration/frameworks/test_tensorflow_unit.py similarity index 100% rename from tests/integration/frameworks/test_tensorflow_v2_unit.py rename to tests/integration/frameworks/test_tensorflow_unit.py diff --git a/tests/unit/_internal/models/test_model.py b/tests/unit/_internal/models/test_model.py index 5f1b9db5304..843046a2414 100644 --- a/tests/unit/_internal/models/test_model.py +++ b/tests/unit/_internal/models/test_model.py @@ -29,7 +29,7 @@ expected_yaml = """\ name: test version: v1 -module: tests.unit._internal.models.test_model +module: test_model labels: label: stringvalue options: diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 99a0972c138..f8f38a74b64 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -145,7 +145,7 @@ def noop_sync(data: str) -> str: # type: ignore @svc.api(input=Text(), output=Text()) def invalid(data: str) -> str: # type: ignore - raise RuntimeError("invalid implementation.") + raise NotImplementedError return svc diff --git a/tests/unit/grpc/interceptors/test_access.py b/tests/unit/grpc/interceptors/test_access.py index b97f0a724c3..32e268706ee 100644 --- a/tests/unit/grpc/interceptors/test_access.py +++ b/tests/unit/grpc/interceptors/test_access.py @@ -1,4 +1,4 @@ -# pylint: disable=unused-argument,used-before-assignment,assignment-from-no-return +# pylint: disable=unused-argument from __future__ import annotations import typing as t diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/utils/_static/chinese.jpg b/tests/utils/_static/chinese.jpg deleted file mode 100644 index 6660aaec7ec..00000000000 Binary files a/tests/utils/_static/chinese.jpg and /dev/null differ diff --git a/tests/utils/_static/detectron2_sample.jpg b/tests/utils/_static/detectron2_sample.jpg deleted file mode 100644 index 9659f0d5e17..00000000000 Binary files a/tests/utils/_static/detectron2_sample.jpg and /dev/null differ diff --git a/tests/utils/frameworks/__init__.py b/tests/utils/frameworks/__init__.py deleted file mode 100644 index ff9569945a5..00000000000 --- a/tests/utils/frameworks/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# ============================================================================== -# Copyright (c) 2021 Atalaya Tech. Inc -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== diff --git a/tests/utils/frameworks/fastai_utils.py b/tests/utils/frameworks/fastai_utils.py deleted file mode 100644 index dfcc9bc977d..00000000000 --- a/tests/utils/frameworks/fastai_utils.py +++ /dev/null @@ -1,71 +0,0 @@ -from __future__ import annotations - -from typing import Any -from typing import TYPE_CHECKING - -import numpy as np -import pandas as pd -import sklearn.datasets as datasets -from fastai.learner import Learner -from fastai.metrics import accuracy -from fastai.data.block import DataBlock -from fastai.torch_core import Module -from fastai.torch_core import set_seed -from fastai.tabular.all import tabular_learner -from fastai.tabular.all import TabularDataLoaders - -from tests.utils.frameworks.pytorch_utils import LinearModel - -if TYPE_CHECKING: - from sklearn.utils import Bunch - - import bentoml._internal.external_typing as ext - -SEED = 123 - -set_seed(SEED, reproducible=True) - -iris: Bunch = datasets.load_iris() -X = pd.DataFrame(iris.data[:, :2], columns=iris.feature_names[:2]) -y = pd.Series(iris.target, name="label") - - -class Loss(Module): - reduction = "none" - - def forward(self, x: Any, _y: Any): - return x - - def activation(self, x: Any): - return x - - def decodes(self, x: Any): - return x - - -# read in data -def tabular_model() -> Learner: - dl = TabularDataLoaders.from_df( - df=pd.concat([X, y], axis=1), - cont_names=list(X.columns), - y_names="label", - num_workers=0, - ) - model = tabular_learner(dl, metrics=accuracy, layers=[3]) - model.fit(1) - return model - - -def get_items(_x: Any) -> ext.NpNDArray: - return np.ones([5, 5], np.float32) - - -def custom_model(): - model = LinearModel() - loss = Loss() - - dblock = DataBlock(get_items=get_items, get_y=np.sum) - dls = dblock.datasets(None).dataloaders() - learner = Learner(dls, model, loss) - learner.fit(1) - return learner diff --git a/tests/utils/frameworks/paddle_utils.py b/tests/utils/frameworks/paddle_utils.py deleted file mode 100644 index 953900f7d28..00000000000 --- a/tests/utils/frameworks/paddle_utils.py +++ /dev/null @@ -1,37 +0,0 @@ -import paddle -import pandas as pd -import paddle.nn as nn -from paddle.static import InputSpec - -IN_FEATURES = 13 -OUT_FEATURES = 1 - -test_df = pd.DataFrame( - [ - [ - -0.0405441, - 0.06636364, - -0.32356227, - -0.06916996, - -0.03435197, - 0.05563625, - -0.03475696, - 0.02682186, - -0.37171335, - -0.21419304, - -0.33569506, - 0.10143217, - -0.21172912, - ] - ] -) - - -class LinearModel(nn.Layer): - def __init__(self): - super(LinearModel, self).__init__() - self.fc = nn.Linear(IN_FEATURES, OUT_FEATURES) - - @paddle.jit.to_static(input_spec=[InputSpec(shape=[IN_FEATURES], dtype="float32")]) - def forward(self, x): - return self.fc(x) diff --git a/tests/utils/frameworks/pytorch_utils.py b/tests/utils/frameworks/pytorch_utils.py deleted file mode 100644 index 58e0cd09ad9..00000000000 --- a/tests/utils/frameworks/pytorch_utils.py +++ /dev/null @@ -1,72 +0,0 @@ -import numpy as np -import torch -import pandas as pd -import torch.nn as nn - -test_df = pd.DataFrame([[1] * 5]) - - -class LinearModel(nn.Module): - def __init__(self): - super().__init__() - self.linear = nn.Linear(5, 1, bias=False) - torch.nn.init.ones_(self.linear.weight) - - def forward(self, x): - return self.linear(x) - - -def make_pytorch_lightning_linear_model_class(): - import pytorch_lightning as pl - - class LightningLinearModel(pl.LightningModule): - def __init__(self): - super().__init__() - self.linear = nn.Linear(5, 1, bias=False) - torch.nn.init.ones_(self.linear.weight) - - def forward(self, x): - return self.linear(x) - - return LightningLinearModel - - -def predict_df(model: nn.Module, df: pd.DataFrame): - input_data = df.to_numpy().astype(np.float32) - input_tensor = torch.from_numpy(input_data) - return model(input_tensor).unsqueeze(dim=0).item() - - -class LinearModelWithBatchAxis(nn.Module): - def __init__(self): - super(LinearModelWithBatchAxis, self).__init__() - self.linear = nn.Linear(5, 1, bias=False) - torch.nn.init.ones_(self.linear.weight) - - def forward(self, x, batch_axis=0): - if batch_axis == 1: - x = x.permute([1, 0]) - res = self.linear(x) - if batch_axis == 1: - res = res.permute([0, 1]) - - return res - - -class ExtendedModel(nn.Module): - def __init__(self, D_in, H, D_out): - """ - In the constructor we instantiate two nn.Linear modules and assign them as - member variables. - """ - super(ExtendedModel, self).__init__() - self.linear1 = nn.Linear(D_in, H) - self.linear2 = nn.Linear(H, D_out) - - def forward(self, x, bias=0.0): - """ - In the forward function we accept a Tensor of input data and an optional bias - """ - h_relu = self.linear1(x).clamp(min=0) - y_pred = self.linear2(h_relu) - return y_pred + bias diff --git a/tests/utils/frameworks/sklearn_utils.py b/tests/utils/frameworks/sklearn_utils.py deleted file mode 100644 index 3a7f10dd0fd..00000000000 --- a/tests/utils/frameworks/sklearn_utils.py +++ /dev/null @@ -1,51 +0,0 @@ -from collections import namedtuple - -import pandas as pd -from sklearn.datasets import load_iris -from sklearn.neighbors import KNeighborsClassifier - -test_data = { - "mean radius": 10.80, - "mean texture": 21.98, - "mean perimeter": 68.79, - "mean area": 359.9, - "mean smoothness": 0.08801, - "mean compactness": 0.05743, - "mean concavity": 0.03614, - "mean concave points": 0.2016, - "mean symmetry": 0.05977, - "mean fractal dimension": 0.3077, - "radius error": 1.621, - "texture error": 2.240, - "perimeter error": 20.20, - "area error": 20.02, - "smoothness error": 0.006543, - "compactness error": 0.02148, - "concavity error": 0.02991, - "concave points error": 0.01045, - "symmetry error": 0.01844, - "fractal dimension error": 0.002690, - "worst radius": 12.76, - "worst texture": 32.04, - "worst perimeter": 83.69, - "worst area": 489.5, - "worst smoothness": 0.1303, - "worst compactness": 0.1696, - "worst concavity": 0.1927, - "worst concave points": 0.07485, - "worst symmetry": 0.2965, - "worst fractal dimension": 0.07662, -} - -test_df = pd.DataFrame([test_data]) - -ModelWithData = namedtuple("ModelWithData", ["model", "data"]) - - -def sklearn_model_data(clf=KNeighborsClassifier, num_data=4) -> ModelWithData: - model = clf() - iris = load_iris() - X = iris.data[:, :num_data] - Y = iris.target - model.fit(X, Y) - return ModelWithData(model=model, data=X)