Updated CircleCI config to use dynamic workflows (OpenLineage#606)

* Updated CircleCI config to use dynamic workflows Signed-off-by: Michael Collado <[email protected]> * Added workflow files and use yq to join Signed-off-by: Michael Collado <[email protected]> * Remove pointer references as they aren't handled by yq Signed-off-by: Michael Collado <[email protected]> * Add support for no-op builds for doc and proposal changes Signed-off-by: Michael Collado <[email protected]> * Updated comments in config.yml file Signed-off-by: Michael Collado <[email protected]> * Added New Integrations section to the CONTRIBUTING.md file Signed-off-by: Michael Collado <[email protected]>
hmoazam · Mar 16, 2022 · 94a3a48 · 94a3a48
1 parent fcc2ea2
commit 94a3a48
Show file tree

Hide file tree

Showing 10 changed files with 748 additions and 572 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml
diff --git a/.circleci/workflows/openlineage-integration-airflow.yml b/.circleci/workflows/openlineage-integration-airflow.yml
@@ -0,0 +1,56 @@
+workflows:
+  openlineage-integration-airflow:
+    jobs:
+      - unit-test-integration-common
+      - build-integration-common:
+          filters:
+            branches:
+              only: main
+          build_tag: ".dev<< pipeline.number >>"
+          requires:
+            - unit-test-integration-common
+      - unit-test-integration-airflow-1
+      - unit-test-integration-airflow-2
+      - integration-test-integration-airflow-1-10:
+          context: integration-tests
+          requires:
+            - unit-test-integration-airflow-1
+            - unit-test-integration-common
+            - unit-test-client-python
+          filters:
+            branches:
+              ignore: /pull\/[0-9]+/
+      - integration-test-integration-airflow:
+          matrix:
+            parameters:
+              airflow-image: ['apache/airflow:2.1.3-python3.7', 'apache/airflow:2.2.4-python3.7']
+          context: integration-tests
+          requires:
+            - unit-test-integration-airflow-2
+            - unit-test-integration-common
+            - unit-test-client-python
+          filters:
+            branches:
+              ignore: /pull\/[0-9]+/
+      - integration-test-integration-airflow-failure:
+          context: integration-tests
+          requires:
+            - unit-test-integration-airflow-2
+            - unit-test-integration-common
+            - unit-test-client-python
+          filters:
+            branches:
+              ignore: /pull\/[0-9]+/
+      - build-integration-airflow:
+          filters:
+            branches:
+              only: main
+          build_tag: ".dev<< pipeline.number >>"
+          requires:
+            - integration-test-integration-airflow
+      - workflow_complete:
+          requires:
+            - build-integration-airflow
+            - integration-test-integration-airflow-failure
+            - integration-test-integration-airflow
+            - integration-test-integration-airflow-1-10
diff --git a/.circleci/workflows/openlineage-integration-dagster.yml b/.circleci/workflows/openlineage-integration-dagster.yml
@@ -0,0 +1,17 @@
+workflows:
+  openlineage-integration-dagster:
+    jobs:
+      - unit-test-integration-dagster:
+          requires:
+            - build-client-python
+      - build-integration-dagster:
+          filters:
+            branches:
+              only: main
+          build_tag: ".dev<< pipeline.number >>"
+          requires:
+            - unit-test-integration-dagster
+      - workflow_complete:
+          requires:
+            - build-integration-dagster
+            - unit-test-integration-dagster
diff --git a/.circleci/workflows/openlineage-integration-dbt.yml b/.circleci/workflows/openlineage-integration-dbt.yml
@@ -0,0 +1,11 @@
+workflows:
+  openlineage-integration-dbt:
+    jobs:
+      - build-integration-dbt:
+          filters:
+            branches:
+              only: main
+          build_tag: ".dev<< pipeline.number >>"
+      - workflow_complete:
+          requires:
+            - build-integration-dbt
diff --git a/.circleci/workflows/openlineage-integration-publish.yml b/.circleci/workflows/openlineage-integration-publish.yml
@@ -0,0 +1,23 @@
+workflows:
+  openlineage-integration-publish:
+    jobs:
+      - publish-snapshot-python:
+          filters:
+            branches:
+              only: main
+          context: release
+          requires:
+            - build-client-python
+            - build-integration-common
+            - build-integration-airflow
+            - build-integration-dbt
+            - build-integration-dagster
+      - publish-spec:
+          filters:
+            branches:
+              only: main
+          context: release
+      - workflow_complete:
+          requires:
+            - publish-snapshot-python
+            - publish-spec
diff --git a/.circleci/workflows/openlineage-integration-python.yml b/.circleci/workflows/openlineage-integration-python.yml
@@ -0,0 +1,15 @@
+workflows:
+  openlineage-integration-python:
+    jobs:
+      - unit-test-client-python
+      - build-client-python:
+          filters:
+            branches:
+              only: main
+          build_tag: ".dev<< pipeline.number >>"
+          requires:
+            - unit-test-client-python
+      - workflow_complete:
+          requires:
+            - build-client-python
+            - unit-test-client-python
diff --git a/.circleci/workflows/openlineage-java.yml b/.circleci/workflows/openlineage-java.yml
@@ -0,0 +1,29 @@
+workflows:
+  openlineage-java:
+    jobs:
+      - build-client-java
+      - publish-snapshot-client-java:
+          context: release
+          requires:
+            - build-client-java
+      - build-integration-spark:
+          matrix:
+            parameters:
+              spark-version: [ '2.4.1', '3.1.2' ]
+      - integration-test-integration-spark:
+          matrix:
+            parameters:
+              spark-version: [ '2.4.1', '3.1.2' ]
+          requires:
+            - build-integration-spark
+      - publish-snapshot-integration-spark:
+          filters:
+            branches:
+              only: main
+          context: release
+          requires:
+            - integration-test-integration-spark
+      - workflow_complete:
+          requires:
+            - publish-snapshot-integration-spark
+            - integration-test-integration-spark
diff --git a/.circleci/workflows/openlineage-proxy.yml b/.circleci/workflows/openlineage-proxy.yml
@@ -0,0 +1,7 @@
+workflows:
+  openlineage-proxy:
+    jobs:
+      - build-proxy-backend
+      - workflow_complete:
+          requires:
+            - build-proxy-backend
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -70,6 +70,22 @@ In the description provide the following sections:
 
 This can be just a couple paragraphs to start with.
 
+## New Integrations
+New integrations should be added under the [./integrations](/integrations) folder. Each module
+should have its own build configuration (e.g., `build.gradle` for a Gradle project, `setup.py` for 
+python, etc.) with appropriate unit tests and integration tests (when possible).
+
+Adding a new integration requires updating the CI build configuration with a new workflow. Job
+definitions, orbs, parameters, etc. shoudl be added to the
+[.circleci/continue_config.yml](`continue_config.yml`) file. Workflow definition files are added to
+the [.circleci/workflows](.circleci/workflows) directory. Each workflow file adheres to the CircleCI
+config.yml schema, including only the workflows subschema (see
+[https://circleci.com/docs/2.0/configuration-reference/#workflows](the CircleCI docs) for the schema
+specification). Each workflow must include a `workflow_complete` job that `requires` each terminal
+required step in the workflow (e.g., you might depend on `run-feature-integration-tests` as the
+final step in the workflow). Job names must be unique across all workflows, as ultimately the
+workflows are merged into a single config file. See existing workflows for examples.
+
 ## First-Time Contributors
 
 If this is your first contribution to open source, you can [follow this tutorial][contributiontutorial] or check [this video series][contributionvideos] to learn about the contribution workflow with GitHub.