feat(ingest): add display name for dynamodb tables (#12534)

datahub-project · Feb 10, 2025 · 2323aff · 2323aff
1 parent 704526b
commit 2323aff
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 3 deletions.
diff --git a/metadata-ingestion/docs/sources/dynamodb/dynamodb_pre.md b/metadata-ingestion/docs/sources/dynamodb/dynamodb_pre.md
@@ -1,6 +1,6 @@
 ### Prerequisities
 
-Notice of breaking change: in the latest version of the DynamoDB connector, `aws_region` is now a required configuration. The connector will no longer loop through all AWS regions; instead, it will only use the region passed into the recipe configuration.
+Notice of breaking change: Starting v0.13.3, `aws_region` is now a required configuration for DynamoDB Connector. The connector will no longer loop through all AWS regions; instead, it will only use the region passed into the recipe configuration.
 
 In order to execute this source, you need to attach the `AmazonDynamoDBReadOnlyAccess` policy to a user in your AWS account. Then create an API access key and secret for the user.
 
@@ -24,3 +24,10 @@ For a user to be able to create API access key, it needs the following access ke
   ]
 }
 ```
+
+### Concept Mapping
+
+| Source Concept | DataHub Concept                                           | Notes |
+| -------------- | --------------------------------------------------------- | ----- |
+| `"dynamodb"`   | [Data Platform](../../metamodel/entities/dataPlatform.md) |       |
+| DynamoDB Table | [Dataset](../../metamodel/entities/dataset.md)            |       |
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py b/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py
@@ -165,6 +165,10 @@ def report_dropped(self, name: str) -> None:
     SourceCapability.PLATFORM_INSTANCE,
     "By default, platform_instance will use the AWS account id",
 )
+@capability(
+    SourceCapability.CLASSIFICATION,
+    "Optionally enabled via `classification.enabled`",
+)
 class DynamoDBSource(StatefulIngestionSourceBase):
     """
     This plugin extracts the following:
@@ -244,6 +248,7 @@ def _process_table(
             name=dataset_name,
         )
         dataset_properties = DatasetPropertiesClass(
+            name=table_name,
             tags=[],
             customProperties={
                 "table.arn": table_info["TableArn"],

diff --git a/...-ingestion/tests/integration/dynamodb/dynamodb_default_platform_instance_mces_golden.json b/...-ingestion/tests/integration/dynamodb/dynamodb_default_platform_instance_mces_golden.json
@@ -36,7 +36,8 @@
                 },
                 {
                     "fieldPath": "city",
-                    "nullable": true,
+                    "nullable": false,
+                    "description": "Sort Key",
                     "type": {
                         "type": {
                             "com.linkedin.schema.StringType": {}
@@ -175,6 +176,7 @@
                 "table.arn": "arn:aws:dynamodb:us-west-2:123456789012:table/Location",
                 "table.totalItems": "1"
             },
+            "name": "Location",
             "tags": []
         }
     },

diff --git a/metadata-ingestion/tests/integration/dynamodb/dynamodb_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dynamodb/dynamodb_platform_instance_mces_golden.json
@@ -36,7 +36,8 @@
                 },
                 {
                     "fieldPath": "city",
-                    "nullable": true,
+                    "nullable": false,
+                    "description": "Sort Key",
                     "type": {
                         "type": {
                             "com.linkedin.schema.StringType": {}
@@ -186,6 +187,7 @@
                 "table.arn": "arn:aws:dynamodb:us-west-2:123456789012:table/Location",
                 "table.totalItems": "1"
             },
+            "name": "Location",
             "tags": []
         }
     },

diff --git a/metadata-ingestion/tests/integration/dynamodb/test_dynamodb.py b/metadata-ingestion/tests/integration/dynamodb/test_dynamodb.py
@@ -29,9 +29,11 @@ def test_dynamodb(pytestconfig, tmp_path):
         TableName="Location",
         KeySchema=[
             {"AttributeName": "partitionKey", "KeyType": "HASH"},
+            {"AttributeName": "city", "KeyType": "RANGE"},
         ],
         AttributeDefinitions=[
             {"AttributeName": "partitionKey", "AttributeType": "S"},
+            {"AttributeName": "city", "AttributeType": "S"},
         ],
         ProvisionedThroughput={"ReadCapacityUnits": 10, "WriteCapacityUnits": 10},
     )