Added support for provider packages for Airflow 2.0 (apache#11487)

* Separate changes/readmes for backport and regular providers

We have now separate release notes for backport provider
packages and regular provider packages.

They have different versioning - backport provider
packages with CALVER, regular provider packages with
semver.

* Added support for provider packages for Airflow 2.0

This change consists of the following changes:

* adds provider package support for 2.0
* adds generation of package readme and change notes
* versions are for now hard-coded to 0.0.1 for first release
* adds automated tests for installation of the packages
* rename backport package readmes/changes to BACKPORT_*
* adds regulaar packge readmes/changes
* updates documentation on generating the provider packaes
* adds CI tests for the packages
* maintains backport packages generation with --backports flag

Fixes apache#11421
Fixes apache#11424

.github/workflows/ci.yml

@@ -271,14 +271,15 @@ jobs:
       - name: "Spell check docs"
         run: ./scripts/ci/docs/ci_docs.sh --spellcheck-only
 
-  prepare-provider-packages:
+  prepare-backport-provider-packages:
     timeout-minutes: 30
     name: "Backport packages"
     runs-on: ubuntu-latest
     needs: [build-info, ci-images]
     env:
       INSTALL_AIRFLOW_VERSION: "1.10.12"
       PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
+      BACKPORT_PACKAGES: "true"
     if: github.repository == 'apache/airflow' || github.event_name != 'schedule'
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -304,7 +305,43 @@ jobs:
         if: always()
         with:
           name: airflow-backport-readmes
-          path: "./files/airflow-backport-readme*"
+          path: "./files/airflow-readme-*"
+
+  prepare-provider-packages:
+    timeout-minutes: 30
+    name: "Provider packages"
+    runs-on: ubuntu-latest
+    needs: [build-info, ci-images]
+    env:
+      INSTALL_AIRFLOW_VERSION: "2.0.0-dev"   # Note that this causes local installation
+      PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
+    if: github.repository == 'apache/airflow' || github.event_name != 'schedule'
+    steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v2
+      - name: "Setup python"
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
+      - name: "Free space"
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+      - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
+      - name: "Prepare & test provider packages"
+        run: ./scripts/ci/provider_packages/ci_prepare_and_test_provider_packages.sh
+      - name: "Upload provider package artifacts"
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: airflow-provider-packages
+          path: "./files/airflow-packages-*"
+      - name: "Upload readme artifacts"
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: airflow-provider-readmes
+          path: "./files/airflow-readme-*"
+
 
   tests-postgres:
     timeout-minutes: 60
@@ -656,6 +693,7 @@ jobs:
       - tests-postgres
       - tests-mysql
       - tests-kubernetes
+      - prepare-backport-provider-packages
       - prepare-provider-packages
       - prod-images
       - docs
@@ -695,7 +733,7 @@ jobs:
       - tests-postgres
       - tests-mysql
       - tests-kubernetes
-      - prepare-provider-packages
+      - prepare-backport-provider-packages
       - ci-images
       - docs
       - docs-spell-check

.pre-commit-config.yaml

@@ -121,7 +121,7 @@ repos:
       - id: insert-license
         name: Add license for all md files
         files: \.md$
-        exclude: ^\.github/.*$|PROVIDERS_CHANGES.*\.md
+        exclude: ^\.github/.*$|BACKPORT_PROVIDERS_CHANGES.*\.md|PROVIDERS_CHANGES.*
         args:
           - --comment-style
           - "<!--|| -->"
@@ -293,8 +293,8 @@ repos:
           (?x)
           ^airflow/providers/apache/cassandra/hooks/cassandra.py$|
           ^airflow/providers/apache/hive/operators/hive_stats.py$|
-          ^airflow/providers/apache/hive/PROVIDERS_CHANGES_*|
-          ^airflow/providers/apache/hive/README.md$|
+          ^airflow/providers/apache/hive/.*PROVIDERS_CHANGES_*|
+          ^airflow/providers/apache/hive/.*README.md$|
           ^tests/providers/apache/cassandra/hooks/test_cassandra.py$|
           ^docs/howto/connection/cassandra.rst$|
           ^CHANGELOG.txt$

.rat-excludes

@@ -78,6 +78,7 @@ rat-results.txt
 apache-airflow-.*\+source.tar.gz.*
 apache-airflow-.*\+bin.tar.gz.*
 PULL_REQUEST_TEMPLATE.md
+BACKPORT_PROVIDERS_CHANGES*.md
 PROVIDERS_CHANGES*.md
 
 # Temporary list of files to make compatible with Pylint

CI.rst

@@ -68,7 +68,7 @@ You can use those variables when you try to reproduce the build locally.
 |                                         |             |             |            | the container. We mount only selected,          |
 |                                         |             |             |            | important folders. We do not mount the whole    |
 |                                         |             |             |            | project folder in order to avoid accidental     |
-|                                         |             |             |            | use of artifacts (such ass ``.egginfo``         |
+|                                         |             |             |            | use of artifacts (such as ``egg-info``          |
 |                                         |             |             |            | directories) generated locally on the           |
 |                                         |             |             |            | host during development.                        |
 +-----------------------------------------+-------------+-------------+------------+-------------------------------------------------+

Dockerfile.ci

@@ -314,7 +314,7 @@ COPY scripts/in_container/entrypoint_ci.sh /entrypoint
 RUN chmod a+x /entrypoint
 
 # We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not
-# copying over stuff that is accidentally generated or that we do not need (such as .egginfo)
+# copying over stuff that is accidentally generated or that we do not need (such as egg-info)
 # if you want to add something that is missing and you expect to see it in the image you can
 # add it with ! in .dockerignore next to the airflow, test etc. directories there
 COPY . ${AIRFLOW_SOURCES}/