feat(ui/ingest): unity-catalog => databricks (#14636)

Author: ligfx

datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx

@@ -105,6 +105,17 @@ enum IngestionSourceType {
 
 const DEFAULT_PAGE_SIZE = 25;
 
+const mapSourceTypeAliases = <T extends { type: string }>(source?: T): T | undefined => {
+    if (source) {
+        let { type } = source;
+        if (type === 'unity-catalog') {
+            type = 'databricks';
+        }
+        return { ...source, type };
+    }
+    return undefined;
+};
+
 const removeExecutionsFromIngestionSource = (source) => {
     if (source) {
         return {
@@ -561,7 +572,7 @@ export const IngestionSourceList = ({ showCreateModal, setShowCreateModal }: Pro
                 </PaginationContainer>
             </SourceContainer>
             <IngestionSourceBuilderModal
-                initialState={removeExecutionsFromIngestionSource(focusSource)}
+                initialState={mapSourceTypeAliases(removeExecutionsFromIngestionSource(focusSource))}
                 open={isBuildingSource}
                 onSubmit={onSubmit}
                 onCancel={onCancel}

datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts

@@ -234,12 +234,12 @@ import {
     AZURE,
     BIGQUERY_BETA,
     CSV,
+    DATABRICKS,
     DBT_CLOUD,
     MYSQL,
     OKTA,
     POWER_BI,
     SAC,
-    UNITY_CATALOG,
     VERTICA,
 } from '@app/ingest/source/builder/constants';
 import { BIGQUERY } from '@app/ingest/source/conf/bigquery/bigquery';
@@ -499,7 +499,7 @@ export const RECIPE_FIELDS: RecipeFields = {
         ],
         filterSectionTooltip: 'Include or exclude specific Schemas, Tables and Views from ingestion.',
     },
-    [UNITY_CATALOG]: {
+    [DATABRICKS]: {
         fields: [WORKSPACE_URL, TOKEN],
         filterFields: [
             UNITY_METASTORE_ID_ALLOW,
@@ -616,11 +616,4 @@ export const RECIPE_FIELDS: RecipeFields = {
 
 export const CONNECTORS_WITH_FORM = new Set(Object.keys(RECIPE_FIELDS));
 
-export const CONNECTORS_WITH_TEST_CONNECTION = new Set([
-    SNOWFLAKE,
-    LOOKER,
-    BIGQUERY_BETA,
-    BIGQUERY,
-    UNITY_CATALOG,
-    SAC,
-]);
+export const CONNECTORS_WITH_TEST_CONNECTION = new Set([SNOWFLAKE, LOOKER, BIGQUERY_BETA, BIGQUERY, DATABRICKS, SAC]);

datahub-web-react/src/app/ingest/source/builder/sources.json

@@ -32,12 +32,12 @@
         "recipe": "source: \n    type: snowflake\n    config:\n        account_id: null\n        include_table_lineage: true\n        include_view_lineage: true\n        include_tables: true\n        include_views: true\n        profiling:\n            enabled: true\n            profile_table_level_only: true\n        stateful_ingestion:\n            enabled: true"
     },
     {
-        "urn": "urn:li:dataPlatform:unity-catalog",
-        "name": "unity-catalog",
+        "urn": "urn:li:dataPlatform:databricks",
+        "name": "databricks",
         "displayName": "Databricks",
         "description": "Import Metastores, Schemas, Tables, lineage, queries, and statistics from Databricks Unity Catalog.",
-        "docsUrl": "https://docs.datahub.com/docs/generated/ingestion/sources/databricks/#module-unity-catalog",
-        "recipe": "source:\n    type: unity-catalog\n    config:\n        # Coordinates\n        workspace_url: null\n        include_table_lineage: true\n        include_column_lineage: false\n        stateful_ingestion:\n            enabled: true"
+        "docsUrl": "https://docs.datahub.com/docs/generated/ingestion/sources/databricks/",
+        "recipe": "source:\n    type: databricks\n    config:\n        # Coordinates\n        workspace_url: null\n        include_table_lineage: true\n        include_column_lineage: false\n        stateful_ingestion:\n            enabled: true"
     },
     {
         "urn": "urn:li:dataPlatform:looker",

datahub-web-react/src/app/ingestV2/source/IngestionSourceList.tsx

@@ -114,6 +114,17 @@ export enum IngestionSourceType {
 
 const DEFAULT_PAGE_SIZE = 25;
 
+const mapSourceTypeAliases = <T extends { type: string }>(source?: T): T | undefined => {
+    if (source) {
+        let { type } = source;
+        if (type === 'unity-catalog') {
+            type = 'databricks';
+        }
+        return { ...source, type };
+    }
+    return undefined;
+};
+
 const removeExecutionsFromIngestionSource = (source) => {
     if (source) {
         return {
@@ -695,7 +706,7 @@ export const IngestionSourceList = ({
                 )}
             </SourceContainer>
             <IngestionSourceBuilderModal
-                initialState={removeExecutionsFromIngestionSource(focusSource)}
+                initialState={mapSourceTypeAliases(removeExecutionsFromIngestionSource(focusSource))}
                 open={showCreateModal}
                 onSubmit={onSubmit}
                 onCancel={onCancel}

datahub-web-react/src/app/ingestV2/source/builder/RecipeForm/constants.ts

@@ -233,12 +233,12 @@ import {
 import {
     AZURE,
     CSV,
+    DATABRICKS,
     DBT_CLOUD,
     MYSQL,
     OKTA,
     POWER_BI,
     SAC,
-    UNITY_CATALOG,
     VERTICA,
 } from '@app/ingestV2/source/builder/constants';
 import { BIGQUERY } from '@app/ingestV2/source/conf/bigquery/bigquery';
@@ -471,7 +471,7 @@ export const RECIPE_FIELDS: RecipeFields = {
         ],
         filterSectionTooltip: 'Include or exclude specific Schemas, Tables and Views from ingestion.',
     },
-    [UNITY_CATALOG]: {
+    [DATABRICKS]: {
         fields: [WORKSPACE_URL, TOKEN],
         filterFields: [
             UNITY_METASTORE_ID_ALLOW,

datahub-web-react/src/app/ingestV2/source/builder/sources.json

@@ -32,12 +32,12 @@
         "recipe": "source: \n    type: snowflake\n    config:\n        account_id: null\n        include_table_lineage: true\n        include_view_lineage: true\n        include_tables: true\n        include_views: true\n        profiling:\n            enabled: true\n            profile_table_level_only: true\n        stateful_ingestion:\n            enabled: true"
     },
     {
-        "urn": "urn:li:dataPlatform:unity-catalog",
-        "name": "unity-catalog",
+        "urn": "urn:li:dataPlatform:databricks",
+        "name": "databricks",
         "displayName": "Databricks",
         "description": "Import Metastores, Schemas, Tables, lineage, queries, and statistics from Databricks Unity Catalog.",
-        "docsUrl": "https://docs.datahub.com/docs/generated/ingestion/sources/databricks/#module-unity-catalog",
-        "recipe": "source:\n    type: unity-catalog\n    config:\n        # Coordinates\n        workspace_url: null\n        include_table_lineage: true\n        include_column_lineage: false\n        stateful_ingestion:\n            enabled: true"
+        "docsUrl": "https://docs.datahub.com/docs/generated/ingestion/sources/databricks/",
+        "recipe": "source:\n    type: databricks\n    config:\n        # Coordinates\n        workspace_url: null\n        include_table_lineage: true\n        include_column_lineage: false\n        stateful_ingestion:\n            enabled: true"
     },
     {
         "urn": "urn:li:dataPlatform:looker",

metadata-ingestion/docs/sources/databricks/README.md

@@ -2,7 +2,7 @@ DataHub supports integration with Databricks ecosystem using a multitude of conn
 
 ## Databricks Unity Catalog (new)
 
-The recently introduced [Unity Catalog](https://www.databricks.com/product/unity-catalog) provides a new way to govern your assets within the Databricks lakehouse. If you have Unity Catalog Enabled Workspace, you can use the `unity-catalog` source (aka `databricks` source, see below for details) to integrate your metadata into DataHub as an alternate to the Hive pathway. This also ingests hive metastore catalog in Databricks and is recommended approach to ingest Databricks ecosystem in DataHub.
+The recently introduced [Unity Catalog](https://www.databricks.com/product/unity-catalog) provides a new way to govern your assets within the Databricks lakehouse. If you have Unity Catalog Enabled Workspace, you can use the `databricks` source (aka `unity-catalog` source, see below for details) to integrate your metadata into DataHub as an alternate to the Hive pathway. This also ingests hive metastore catalog in Databricks and is recommended approach to ingest Databricks ecosystem in DataHub.
 
 ## Databricks Hive (old)
 

metadata-ingestion/docs/sources/databricks/unity-catalog_recipe.yml

@@ -1,5 +1,5 @@
 source:
-  type: unity-catalog
+  type: databricks
   config:
     workspace_url: https://my-workspace.cloud.databricks.com
     token: "<token>"