Merge large-requirements.txt and small-requirements.txt (mlflow#5942)

* remove large-requirements.txt

Signed-off-by: harupy <[email protected]>

* rename

Signed-off-by: harupy <[email protected]>

* Remove large-requirements.txt

Signed-off-by: harupy <[email protected]>

* fix env vars

Signed-off-by: harupy <[email protected]>

* remove scipy

Signed-off-by: harupy <[email protected]>

* remove pyspark in test-requirements

Signed-off-by: harupy <[email protected]>

* install pyspark in pyfunc job

Signed-off-by: harupy <[email protected]>

* remove redundant env var

Signed-off-by: harupy <[email protected]>

* fix skinny job

Signed-off-by: harupy <[email protected]>

* fix typo

Signed-off-by: harupy <[email protected]>

* fix comment

Signed-off-by: harupy <[email protected]>

Author: harupy

.dockerignore

@@ -8,8 +8,7 @@ examples
 
 dev
 # required for Docker build
-!requirements/small-requirements.txt
-!requirements/large-requirements.txt
+!requirements/test-requirements.txt
 !requirements/lint-requirements.txt
 !requirements/extra-ml-requirements.txt
 

.github/workflows/cross-version-tests.yml

@@ -152,7 +152,7 @@ jobs:
           python --version
           pip install --upgrade pip wheel
           pip install -e .
-          pip install -r requirements/small-requirements.txt
+          pip install -r requirements/test-requirements.txt
       - name: Install ${{ matrix.package }} ${{ matrix.version }}
         env:
           CACHE_DIR: /home/runner/.cache/wheels

.github/workflows/examples.yml

@@ -69,8 +69,7 @@ jobs:
     - name: Install dependencies
       if: ${{ github.event_name == 'schedule' || steps.check-diff.outputs.examples_changed == 'true' }}
       env:
-        INSTALL_SMALL_PYTHON_DEPS: true
-        INSTALL_LARGE_PYTHON_DEPS: true
+        INSTALL_ML_DEPENDENCIES: true
       run: |
         source ./dev/install-common-deps.sh
 

.github/workflows/master.yml

@@ -55,8 +55,7 @@ jobs:
         echo "::add-matcher::.github/workflows/matchers/pylint.json"
     - name: Install dependencies
       env:
-        INSTALL_LARGE_PYTHON_DEPS: true
-        INSTALL_SMALL_PYTHON_DEPS: true
+        INSTALL_ML_DEPENDENCIES: true
       run: |
         source ./dev/install-common-deps.sh
         pip install -r requirements/lint-requirements.txt
@@ -110,8 +109,7 @@ jobs:
     - uses: ./.github/actions/cache-pip
     - name: Install dependencies
       env:
-        INSTALL_LARGE_PYTHON_DEPS: true
-        INSTALL_SMALL_PYTHON_DEPS: true
+        INSTALL_ML_DEPENDENCIES: true
       run: |
         source ./dev/install-common-deps.sh
     - name: Run tests
@@ -221,8 +219,7 @@ jobs:
       - uses: ./.github/actions/cache-pip
       - name: Install dependencies
         env:
-          INSTALL_LARGE_PYTHON_DEPS: true
-          INSTALL_SMALL_PYTHON_DEPS: true
+          INSTALL_ML_DEPENDENCIES: true
         run: |
           source ./dev/install-common-deps.sh
       - name: Run tests
@@ -249,9 +246,6 @@ jobs:
           java-version: 11
           distribution: 'adopt'
       - name: Install dependencies
-        env:
-          INSTALL_SMALL_PYTHON_DEPS: true
-          INSTALL_LARGE_PYTHON_DEPS: false
         run: |
           source ./dev/install-common-deps.sh
           pip install pyspark
@@ -276,12 +270,9 @@ jobs:
           java-version: 11
           distribution: 'adopt'
       - name: Install dependencies
-        env:
-          INSTALL_SMALL_PYTHON_DEPS: true
-          INSTALL_LARGE_PYTHON_DEPS: false
         run: |
           source ./dev/install-common-deps.sh
-          pip install keras tensorflow
+          pip install keras tensorflow pyspark
       - name: Run tests
         env:
           SPARK_LOCAL_IP: 127.0.0.1
@@ -303,8 +294,7 @@ jobs:
           distribution: 'adopt'
       - name: Install dependencies
         env:
-          INSTALL_LARGE_PYTHON_DEPS: true
-          INSTALL_SMALL_PYTHON_DEPS: true
+          INSTALL_ML_DEPENDENCIES: true
         run: |
           source ./dev/install-common-deps.sh
       - name: Run tests
@@ -328,9 +318,10 @@ jobs:
           distribution: 'adopt'
       - name: Install python dependencies
         run: |
-          pip install -r requirements/small-requirements.txt
+          pip install -r requirements/test-requirements.txt
           pip install --no-dependencies tests/resources/mlflow-test-plugin
           pip install -e .[extras]
+          pip install pyspark
       - name: Run python tests
         run: |
           pytest --ignore-flavors --ignore=tests/projects --ignore=tests/examples tests

CONTRIBUTING.rst

@@ -435,7 +435,7 @@ If you are adding new framework flavor support, you'll need to modify ``pytest``
   a. Add your tests to the ignore list, where the other frameworks are ignored
   b. Add a pytest command for your tests along with the other framework tests (as a separate command to avoid OOM issues)
 
-2. ``requirements/large-requirements.txt``: add your framework and version to the list of requirements
+2. ``requirements/test-requirements.txt``: add your framework and version to the list of requirements
 
 You can see an example of a `flavor PR <https://github.com/mlflow/mlflow/pull/2136/files>`_.
 

EXTRA_DEPENDENCIES.rst

@@ -15,5 +15,4 @@ The full set of extra dependencies are documented, along with the modules that d
 in the following files:
 
 * extra-ml-requirements.txt: ML libraries needed to use model persistence and inference APIs
-* small-requirements.txt, large-requirements.txt: Libraries required to use non-default
-  artifact-logging and tracking server configurations
+* test-requirements.txt: Libraries required to use non-default artifact-logging and tracking server configurations

dev/dev-env-setup.sh

@@ -176,13 +176,13 @@ fi
 echo "The top-level dependencies that will be installed are: "
 
 if [[ -n "$full" ]]; then
-  files=("$rd/small-requirements.txt" "$rd/lint-requirements.txt" "$rd/large-requirements.txt" "$rd/doc-requirements.txt" "$rd/extra-ml-requirements.txt")
+  files=("$rd/test-requirements.txt" "$rd/lint-requirements.txt" "$rd/doc-requirements.txt" "$rd/extra-ml-requirements.txt")
   echo "Files:"
   echo "MLflow test plugin: $MLFLOW_HOME/tests/resources/mlflow-test-plugin"
   echo "The local development branch of MLflow installed in editable mode with 'extras' requirements"
   echo "The following packages: "
 else
-  files=("$rd/small-requirements.txt" "$rd/lint-requirements.txt" "$rd/large-requirements.txt" "$rd/doc-requirements.txt")
+  files=("$rd/test-requirements.txt" "$rd/lint-requirements.txt" "$rd/doc-requirements.txt")
 fi
 tail -n +1 "${files[@]}" | grep "^[^#= ]" | sort | cat
 
@@ -235,7 +235,7 @@ if [[ -n "$full" ]]; then
   pip install $(quiet_command) -e "$MLFLOW_HOME/tests/resources//mlflow-test-plugin"
   echo "Finished installing pip dependencies."
 else
-  files=("$rd/small-requirements.txt" "$rd/lint-requirements.txt" "$rd/large-requirements.txt" "$rd/doc-requirements.txt")
+  files=("$rd/test-requirements.txt" "$rd/lint-requirements.txt" "$rd/doc-requirements.txt")
   for r in "${files[@]}";
   do
     pip install $(quiet_command) -r "$r"

dev/install-common-deps.sh

@@ -29,18 +29,12 @@ export MLFLOW_HOME=$(pwd)
 
 req_files=""
 # Install Python test dependencies only if we're running Python tests
-if [[ "$INSTALL_SMALL_PYTHON_DEPS" == "true" ]]; then
-  # When downloading large packages from PyPI, the connection is sometimes aborted by the
-  # remote host. See https://github.com/pypa/pip/issues/8510.
-  # As a workaround, we retry installation of large packages.
-  req_files+=" -r requirements/small-requirements.txt"
-fi
 if [[ "$INSTALL_SKINNY_PYTHON_DEPS" == "true" ]]; then
   req_files+=" -r requirements/skinny-requirements.txt"
+else
+  req_files+=" -r requirements/test-requirements.txt"
 fi
-if [[ "$INSTALL_LARGE_PYTHON_DEPS" == "true" ]]; then
-  req_files+=" -r requirements/large-requirements.txt"
-
+if [[ "$INSTALL_ML_DEPENDENCIES" == "true" ]]; then
   # Install prophet's dependencies beforehand, otherwise pip would fail to build a wheel for prophet
   if [[ -z "$(pip cache list prophet --format abspath)" ]]; then
     tmp_dir=$(mktemp -d)

dev/setup-spark-datasource-autologging.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 # Test Spark autologging against the Spark 3.0 preview. This script is temporary and should be
 # removed once Spark 3.0 is released in favor of simply updating all tests to run against Spark 3.0
-# (i.e. updating the pyspark dependency version in requirements/large-requirements.txt)
 set -ex
 
 # Build Java package

requirements/dev-requirements.txt

@@ -1,5 +1,4 @@
--r small-requirements.txt
--r large-requirements.txt
+-r test-requirements.txt
 -r extra-ml-requirements.txt
 -r lint-requirements.txt
 -r doc-requirements.txt

requirements/extra-ml-requirements.txt

@@ -33,6 +33,7 @@ h2o
 # Required by mlflow.onnx
 onnx>=1.11.0
 onnxruntime
+tf2onnx
 # Required by mlflow.spark
 pyspark
 # Required by mlflow.shap

requirements/large-requirements.txt

@@ -1,7 +1,6 @@
-## Small test reqs
+## Dependencies required to run tests
 # Required for testing utilities for parsing pip requirements
 pip>=20.1
-scipy
 # NB: We're specifying a test-only minimum version bound for sqlalchemy in order to reliably
 # execute schema consistency checks, the semantics of which were changed in sqlalchemy 1.3.21
 sqlalchemy>=1.3.21
@@ -17,4 +16,3 @@ pillow
 matplotlib
 shap>=0.40
 plotly
-pyspark