From 7923c77e308bda9725d169174de8e31b9cf79853 Mon Sep 17 00:00:00 2001 From: Michael Maltese Date: Tue, 2 Sep 2025 12:56:16 -0400 Subject: [PATCH 1/3] feat(ui/ingest): unity-catalog => databricks - Change newly-creted Databricks recipes to have `type: databricks` - Support visual RecipeForm for both `databricks` and `unity-catalog` (including platform name and link to documentation at the top). --- .../src/app/ingest/source/IngestionSourceList.tsx | 13 ++++++++++++- .../ingest/source/builder/RecipeForm/constants.ts | 13 +++---------- .../src/app/ingest/source/builder/sources.json | 8 ++++---- .../src/app/ingestV2/source/IngestionSourceList.tsx | 13 ++++++++++++- .../ingestV2/source/builder/RecipeForm/constants.ts | 4 ++-- .../src/app/ingestV2/source/builder/sources.json | 8 ++++---- 6 files changed, 37 insertions(+), 22 deletions(-) diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx index 34353afe6c6342..a88ccbe203d866 100644 --- a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx +++ b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx @@ -105,6 +105,17 @@ enum IngestionSourceType { const DEFAULT_PAGE_SIZE = 25; +const mapSourceTypeAliases = (source) => { + if (source) { + let { type } = source; + if (type === 'unity-catalog') { + type = 'databricks'; + } + return { ...source, type }; + } + return undefined; +}; + const removeExecutionsFromIngestionSource = (source) => { if (source) { return { @@ -561,7 +572,7 @@ export const IngestionSourceList = ({ showCreateModal, setShowCreateModal }: Pro { + if (source) { + let { type } = source; + if (type === 'unity-catalog') { + type = 'databricks'; + } + return { ...source, type }; + } + return undefined; +}; + const removeExecutionsFromIngestionSource = (source) => { if (source) { return { @@ -695,7 +706,7 @@ export const IngestionSourceList = ({ )} Date: Tue, 2 Sep 2025 12:39:54 -0400 Subject: [PATCH 2/3] feat(docs/ingest): change Databricks example receipe type unity-catalog => databricks --- metadata-ingestion/docs/sources/databricks/README.md | 2 +- .../docs/sources/databricks/unity-catalog_recipe.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/docs/sources/databricks/README.md b/metadata-ingestion/docs/sources/databricks/README.md index ff638fbd56d45e..47c776c12dcfb7 100644 --- a/metadata-ingestion/docs/sources/databricks/README.md +++ b/metadata-ingestion/docs/sources/databricks/README.md @@ -2,7 +2,7 @@ DataHub supports integration with Databricks ecosystem using a multitude of conn ## Databricks Unity Catalog (new) -The recently introduced [Unity Catalog](https://www.databricks.com/product/unity-catalog) provides a new way to govern your assets within the Databricks lakehouse. If you have Unity Catalog Enabled Workspace, you can use the `unity-catalog` source (aka `databricks` source, see below for details) to integrate your metadata into DataHub as an alternate to the Hive pathway. This also ingests hive metastore catalog in Databricks and is recommended approach to ingest Databricks ecosystem in DataHub. +The recently introduced [Unity Catalog](https://www.databricks.com/product/unity-catalog) provides a new way to govern your assets within the Databricks lakehouse. If you have Unity Catalog Enabled Workspace, you can use the `databricks` source (aka `unity-catalog` source, see below for details) to integrate your metadata into DataHub as an alternate to the Hive pathway. This also ingests hive metastore catalog in Databricks and is recommended approach to ingest Databricks ecosystem in DataHub. ## Databricks Hive (old) diff --git a/metadata-ingestion/docs/sources/databricks/unity-catalog_recipe.yml b/metadata-ingestion/docs/sources/databricks/unity-catalog_recipe.yml index b1d97bacd1cac5..5eb5e4e77ada1e 100644 --- a/metadata-ingestion/docs/sources/databricks/unity-catalog_recipe.yml +++ b/metadata-ingestion/docs/sources/databricks/unity-catalog_recipe.yml @@ -1,5 +1,5 @@ source: - type: unity-catalog + type: databricks config: workspace_url: https://my-workspace.cloud.databricks.com token: "" From b80d6ccba2e6267b74943d1be4104629739ce7a5 Mon Sep 17 00:00:00 2001 From: Michael Maltese Date: Wed, 3 Sep 2025 08:52:08 -0400 Subject: [PATCH 3/3] chore(ui/ingest): add type annotations to mapSourceTypeAliases --- datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx | 2 +- .../src/app/ingestV2/source/IngestionSourceList.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx index a88ccbe203d866..18d8617ff6c084 100644 --- a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx +++ b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx @@ -105,7 +105,7 @@ enum IngestionSourceType { const DEFAULT_PAGE_SIZE = 25; -const mapSourceTypeAliases = (source) => { +const mapSourceTypeAliases = (source?: T): T | undefined => { if (source) { let { type } = source; if (type === 'unity-catalog') { diff --git a/datahub-web-react/src/app/ingestV2/source/IngestionSourceList.tsx b/datahub-web-react/src/app/ingestV2/source/IngestionSourceList.tsx index 38935a98c5fa07..f395d8ec8cd7fd 100644 --- a/datahub-web-react/src/app/ingestV2/source/IngestionSourceList.tsx +++ b/datahub-web-react/src/app/ingestV2/source/IngestionSourceList.tsx @@ -114,7 +114,7 @@ export enum IngestionSourceType { const DEFAULT_PAGE_SIZE = 25; -const mapSourceTypeAliases = (source) => { +const mapSourceTypeAliases = (source?: T): T | undefined => { if (source) { let { type } = source; if (type === 'unity-catalog') {