Split tests to more sub-types (apache#11402)

We seem to have a problem with running all tests at once - most likely due to some resource problems in our CI, therefore it makes sense to split the tests into more batches. This is not yet full implementation of selective tests but it is going in this direction by splitting to Core/Providers/API/CLI tests. The full selective tests approach will be implemented as part of apache#10507 issue. This split is possible thanks to apache#10422 which moved building image to a separate workflow - this way each image is only built once and it is uploaded to a shared registry, where it is quickly downloaded from rather than built by all the jobs separately - this way we can have many more jobs as there is very little per-job overhead before the tests start runnning.
pedresnyman · Oct 11, 2020 · 5bc5994 · 5bc5994
1 parent 4de8f85
commit 5bc5994
Show file tree

Hide file tree

Showing 35 changed files with 401 additions and 119 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -43,7 +43,7 @@ env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   GITHUB_REGISTRY_PULL_IMAGE_TAG: "${{ github.run_id }}"
   GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
-  TEST_TYPES: '["Core", "Integration", "Heisentests"]'
+  TEST_TYPES: '["Core", "Providers", "API", "CLI", "Integration", "Other", "WWW", "Heisentests"]'
 
   # You can switch between building the image in "Build Images" workflow or building them in CI workflow
   # Separately for each job.
@@ -79,7 +79,7 @@ jobs:
       postgresVersions: ${{ steps.versions.outputs.postgres-versions }}
       defaultPostgresVersion: ${{ steps.versions.outputs.default-postgres-version }}
       mysqlVersions: ${{ steps.versions.outputs.mysql-versions }}
-      defaultMysqlVersion: ${{ steps.versions.outputs.default-mysql-version }}
+      defaultMySQLVersion: ${{ steps.versions.outputs.default-mysql-version }}
       helmVersions: ${{ steps.versions.outputs.helm-versions }}
       defaultHelmVersion: ${{ steps.versions.outputs.default-helm-version }}
       kindVersions: ${{ steps.versions.outputs.kind-versions }}
@@ -369,6 +369,12 @@ jobs:
         postgres-version: ${{ fromJson(needs.build-info.outputs.postgresVersions) }}
         test-type: ${{ fromJson(needs.build-info.outputs.testTypes) }}
         exclude: ${{ fromJson(needs.build-info.outputs.postgresExclude) }}
+        # Additionally include all postgres-only tests but only for default versions
+        # in case the regular tests are excluded in the future
+        include:
+          - python_version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+            postgres-version: ${{ needs.build-info.outputs.defaultPostgresVersion }}
+            test-type: "Postgres"
       fail-fast: false
     env:
       BACKEND: postgres
@@ -416,6 +422,12 @@ jobs:
         mysql-version: ${{ fromJson(needs.build-info.outputs.mysqlVersions) }}
         test-type: ${{ fromJson(needs.build-info.outputs.testTypes) }}
         exclude: ${{ fromJson(needs.build-info.outputs.mysqlExclude) }}
+        # Additionally include all mysql-only tests but only for default versions
+        # in case the regular tests are excluded in the future
+        include:
+          - python_version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+            postgres-version: ${{ needs.build-info.outputs.defaultMySQLVersion }}
+            test-type: "MySQL"
       fail-fast: false
     env:
       BACKEND: mysql
@@ -500,9 +512,16 @@ jobs:
     runs-on: ubuntu-latest
     continue-on-error: true
     needs: [build-info, trigger-tests, ci-images]
+    strategy:
+      matrix:
+        include:
+          - backend: mysql
+          - backend: postgres
+          - backend: sqlite
     env:
-      PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
-      BACKEND: postgres
+      BACKEND: ${{ matrix.backend }}
+      PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+      MYSQL_VERSION: ${{needs.build-info.outputs.defaultMySQLVersion}}
       POSTGRES_VERSION: ${{needs.build-info.outputs.defaultPostgresVersion}}
       RUN_TESTS: true
       TEST_TYPE: Quarantined

diff --git a/BREEZE.rst b/BREEZE.rst
@@ -1990,7 +1990,12 @@ This is the current syntax for  `./breeze <./breeze>`_:
 
   Flags:
 
-  Run 'breeze flags' to see all applicable flags.
+  --test-type TEST_TYPE
+          Type of the test to run. One of:
+
+                 All,Core,Providers,API,CLI,Integration,Other,WWW,Heisentests,Postgres,MySQL
+
+          Default: All
 
 
   ####################################################################################################
@@ -2338,6 +2343,16 @@ This is the current syntax for  `./breeze <./breeze>`_:
 
           Default: latest.
 
+  ****************************************************************************************************
+   Flags for running tests
+
+  --test-type TEST_TYPE
+          Type of the test to run. One of:
+
+                 All,Core,Providers,API,CLI,Integration,Other,WWW,Heisentests,Postgres,MySQL
+
+          Default: All
+
   ****************************************************************************************************
    Flags for generation of the backport packages
 

diff --git a/TESTING.rst b/TESTING.rst
@@ -131,23 +131,70 @@ Running Tests for a Specified Target Using Breeze from the Host
 ---------------------------------------------------------------
 
 If you wish to only run tests and not to drop into shell, apply the
-``tests`` command. You can add extra targets and pytest flags after the ``tests`` command.
+``tests`` command. You can add extra targets and pytest flags after the ``--`` command. Note that
+often you want to run the tests with a clean/reset db, so usually you want to add ``--db-reset`` flag
+to breeze.
 
 .. code-block:: bash
 
-     ./breeze tests tests/hooks/test_druid_hook.py tests/tests_core.py --logging-level=DEBUG
+     ./breeze tests tests/hooks/test_druid_hook.py tests/tests_core.py --db-reset -- --logging-level=DEBUG
 
-You can run the whole test suite with a 'tests' test target:
+You can run the whole test suite without adding the test target:
 
 .. code-block:: bash
 
-    ./breeze tests tests
+    ./breeze tests --db-reset
 
 You can also specify individual tests or a group of tests:
 
 .. code-block:: bash
 
-    ./breeze tests tests/test_core.py::TestCore
+    ./breeze tests --db-reset tests/test_core.py::TestCore
+
+
+Running Tests of a specified type from the Host
+-----------------------------------------------
+
+You can also run tests for a specific test type. For the stability and performance point of view
+we separated tests to different test types so that they can be run separately.
+
+You can select the test type by adding ``--test-type TEST_TYPE`` before the test command. There are two
+kinds of test types:
+
+* Per-directories types are added to select subset of the tests based on sub-directories in ``tests`` folder.
+  Example test types there - Core, Providers, CLI. The only action that happens when you choose the right
+  test folders are pre-selected. For those types of tests it is only useful to choose the test type
+  when you do not specify test to run.
+
+Runs all core tests:
+
+.. code-block:: bash
+
+     ./breeze --test-type Core  --db-reset tests
+
+Runs all provider tests:
+
+.. code-block:: bash
+
+     ./breeze --test-type Providers --db-reset tests
+
+* Special kinds of tests - Integration, Heisentests, Quarantined, Postgres, MySQL which are marked with pytest
+  marks and for those you need to select the type using test-type switch. If you want to run such tests
+  using breeze, you need to pass appropriate ``--test-type`` otherwise the test will be skipped.
+  Similarly to the per-directory tests if you do not specify the test or tests to run,
+  all tests of a given type are run
+
+Run quarantined test_task_command.py test:
+
+.. code-block:: bash
+
+     ./breeze --test-type Quarantined tests tests/cli/commands/test_task_command.py --db-reset
+
+Run all Quarantined tests:
+
+.. code-block:: bash
+
+     ./breeze --test-type Quarantined tests --db-reset
 
 
 Airflow Integration Tests

diff --git a/breeze b/breeze
@@ -1072,6 +1072,7 @@ function breeze::parse_arguments() {
             export SKIP_BUILDING_PROD_IMAGE="true"
             export MOUNT_LOCAL_SOURCES="false"
             export SKIP_CHECK_REMOTE_IMAGE="true"
+            export FAIL_ON_GITHUB_DOCKER_PULL_ERROR="true"
             shift 2
             ;;
         --init-script)
@@ -1128,6 +1129,12 @@ function breeze::parse_arguments() {
             echo
             shift
             ;;
+        --test-type)
+            export TEST_TYPE="${2}"
+            echo "Selected test type: ${TEST_TYPE}"
+            echo
+            shift 2
+            ;;
         --)
             shift
             break
@@ -1288,12 +1295,8 @@ function breeze::parse_arguments() {
             ;;
         tests)
             last_subcommand="${1}"
-            if [[ $# -lt 2 ]]; then
-                run_help="true"
-            else
-                shift
-            fi
             command_to_run="run_tests"
+            shift
             ;;
         toggle-suppress-cheatsheet)
             last_subcommand="${1}"
@@ -1437,6 +1440,11 @@ function breeze::prepare_formatted_versions() {
     FORMATTED_DEFAULT_PROD_EXTRAS=$(echo "${DEFAULT_PROD_EXTRAS=}" |
         tr ',' ' ' | fold -w "${indented_screen_width}" -s | sed "s/ /,/g; s/^/${list_prefix}/")
     readonly FORMATTED_DEFAULT_PROD_EXTRAS
+
+    FORMATTED_TEST_TYPES=$(echo "${_breeze_allowed_test_types=""}" |
+        tr ',' ' ' | fold -w "${indented_screen_width}" -s | sed "s/ /,/g; s/^/${list_prefix}/")
+    readonly FORMATTED_TEST_TYPES
+
 }
 
 #######################################################################################################
@@ -1835,7 +1843,7 @@ ${CMDNAME} tests [FLAGS] [TEST_TARGET ..] [-- <EXTRA_ARGS>]
       '${CMDNAME} tests tests
 
 Flags:
-$(breeze::flag_footer)
+$(breeze::flag_tests)
 "
     readonly DETAILED_USAGE_TESTS
     export DETAILED_USAGE_TOGGLE_SUPPRESS_CHEATSHEET="
@@ -2477,6 +2485,25 @@ function breeze::flag_start_airflow() {
 "
 }
 
+#####################################################################################################
+#
+# Prints flags that control tests
+#
+# Outputs:
+#    Flag information.
+#######################################################################################################
+function breeze::flag_tests() {
+    echo "
+--test-type TEST_TYPE
+        Type of the test to run. One of:
+
+${FORMATTED_TEST_TYPES}
+
+        Default: ${_breeze_default_test_type:=}
+
+"
+}
+
 #######################################################################################################
 #
 # Prints all flags
@@ -2538,6 +2565,10 @@ $(breeze::print_star_line)
  Flags for pulling/pushing Docker images (both CI and production)
 $(breeze::flag_pull_push_docker_images)
 
+$(breeze::print_star_line)
+ Flags for running tests
+$(breeze::flag_tests)
+
 $(breeze::print_star_line)
  Flags for generation of the backport packages
 $(breeze::flag_version_suffix)
@@ -2698,6 +2729,7 @@ function breeze::check_and_save_all_params() {
         fi
     fi
 
+
     parameters::check_and_save_allowed_param "BACKEND" "backend" "--backend"
     parameters::check_and_save_allowed_param "KUBERNETES_MODE" "Kubernetes mode" "--kubernetes-mode"
     parameters::check_and_save_allowed_param "KUBERNETES_VERSION" "Kubernetes version" "--kubernetes-version"
@@ -2706,6 +2738,8 @@ function breeze::check_and_save_all_params() {
     parameters::check_and_save_allowed_param "POSTGRES_VERSION" "Postgres version" "--postgres-version"
     parameters::check_and_save_allowed_param "MYSQL_VERSION" "Mysql version" "--mysql-version"
 
+    parameters::check_allowed_param TEST_TYPE "Type of tests" "--test-type"
+
     # Can't verify those - they can be anything, so let's just save them
     parameters::save_to_file DOCKERHUB_USER
     parameters::save_to_file DOCKERHUB_REPO

diff --git a/breeze-complete b/breeze-complete
@@ -33,6 +33,7 @@ _breeze_allowed_kind_versions="v0.8.0"
 _breeze_allowed_mysql_versions="5.7 8"
 _breeze_allowed_postgres_versions="9.6 10"
 _breeze_allowed_kind_operations="start stop restart status deploy test shell"
+_breeze_allowed_test_types="All Core Providers API CLI Integration Other WWW Heisentests Postgres MySQL"
 
 # shellcheck disable=SC2034
 {
@@ -44,6 +45,7 @@ _breeze_allowed_kind_operations="start stop restart status deploy test shell"
     _breeze_default_kind_version=$(echo "${_breeze_allowed_kind_versions}" | awk '{print $1}')
     _breeze_default_postgres_version=$(echo "${_breeze_allowed_postgres_versions}" | awk '{print $1}')
     _breeze_default_mysql_version=$(echo "${_breeze_allowed_mysql_versions}" | awk '{print $1}')
+    _breeze_default_test_type=$(echo "${_breeze_allowed_test_types}" | awk '{print $1}')
 }
 
 _breeze_allowed_install_airflow_versions=$(cat <<-EOF
@@ -156,6 +158,7 @@ dev-apt-deps: additional-dev-apt-deps: dev-apt-command: additional-dev-apt-comma
 runtime-apt-deps: additional-runtime-apt-deps: runtime-apt-command: additional-runtime-apt-command: additional-runtime-apt-env:
 load-default-connections load-example-dags
 install-wheels no-rbac-ui
+test-type:
 "
 
 _breeze_commands="
@@ -180,7 +183,8 @@ kind-cluster
 prepare-backport-readme
 prepare-backport-packages
 static-check
-tests"
+tests
+"
 
 _breeze_help_commands="
 flags
@@ -265,6 +269,12 @@ function breeze_complete::get_known_values_breeze() {
     kind-cluster)
         _breeze_known_values="${_breeze_allowed_kind_operations}"
         ;;
+    tests)
+        _breeze_known_values="$(find tests -name '*.py')"
+        ;;
+    --test-type)
+        _breeze_known_values="${_breeze_allowed_test_types}"
+        ;;
     *)
         _breeze_known_values=""
         ;;

diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml
@@ -39,9 +39,7 @@ services:
       - ENABLE_KIND_CLUSTER
       - ENABLED_INTEGRATIONS
       - RUN_INTEGRATION_TESTS
-      - ONLY_RUN_LONG_RUNNING_TESTS
-      - ONLY_RUN_QUARANTINED_TESTS
-      - ONLY_RUN_HEISEN_TESTS
+      - TEST_TYPE
       - GITHUB_TOKEN
       - GITHUB_REPOSITORY
       - ISSUE_ID
@@ -64,6 +62,9 @@ services:
       - HOST_OS
       - PYTHONDONTWRITEBYTECODE
       - INIT_SCRIPT_FILE
+      - GITHUB_REGISTRY_PULL_IMAGE_TAG
+      - POSTGRES_VERSION
+      - MYSQL_VERSION
     volumes:
       # Pass docker to inside of the container so that Kind and Moto tests can use it.
       - /var/run/docker.sock:/var/run/docker.sock

diff --git a/scripts/ci/libraries/_docker.env b/scripts/ci/libraries/_docker.env
@@ -24,9 +24,13 @@ HOST_OS
 HOST_HOME
 HOST_AIRFLOW_SOURCES
 PYTHON_MAJOR_MINOR_VERSION
+BACKEND
 VERSION_SUFFIX_FOR_PYPI
 VERSION_SUFFIX_FOR_SVN
 PRINT_INFO_FROM_SCRIPTS
 CI
 LOAD_DEFAULT_CONNECTIONS
 LOAD_EXAMPLES
+GITHUB_REGISTRY_PULL_IMAGE_TAG
+POSTGRES_VERSION
+MYSQL_VERSION