From e0633da56bc66ee0dae1560419ad0dcacb10c4da Mon Sep 17 00:00:00 2001 From: Shuixi Li Date: Fri, 14 Mar 2025 13:09:58 -0400 Subject: [PATCH 1/2] add now chart and dataset changed_on use now for chart --- .../src/datahub/ingestion/source/superset.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/superset.py b/metadata-ingestion/src/datahub/ingestion/source/superset.py index 3340dae42bea18..ceb5b74ce56126 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/superset.py +++ b/metadata-ingestion/src/datahub/ingestion/source/superset.py @@ -401,8 +401,11 @@ def construct_dashboard_from_api_data( ) modified_actor = f"urn:li:corpuser:{self.owner_info.get((dashboard_data.get('changed_by') or {}).get('id', -1), 'unknown')}" + + now = datetime.now().strftime("%I:%M%p on %B %d, %Y") + modified_ts = int( - dp.parse(dashboard_data.get("changed_on_utc", "now")).timestamp() * 1000 + dp.parse(dashboard_data.get("changed_on", now)).timestamp() * 1000 ) title = dashboard_data.get("dashboard_title", "") # note: the API does not currently supply created_by usernames due to a bug @@ -514,8 +517,9 @@ def construct_chart_from_chart_data(self, chart_data: dict) -> ChartSnapshot: ) modified_actor = f"urn:li:corpuser:{self.owner_info.get((chart_data.get('changed_by') or {}).get('id', -1), 'unknown')}" + modified_ts = int( - dp.parse(chart_data.get("changed_on_utc", "now")).timestamp() * 1000 + dp.parse(datetime.now().strftime("%I:%M%p on %B %d, %Y")).timestamp() * 1000 ) title = chart_data.get("slice_name", "") @@ -782,8 +786,11 @@ def construct_dataset_from_dataset_data( dataset_url = f"{self.config.display_uri}{dataset_response.get('result', {}).get('url', '')}" modified_actor = f"urn:li:corpuser:{self.owner_info.get((dataset_data.get('changed_by') or {}).get('id', -1), 'unknown')}" + + now = datetime.now().strftime("%I:%M%p on %B %d, %Y") + modified_ts = int( - dp.parse(dataset_data.get("changed_on_utc", "now")).timestamp() * 1000 + dp.parse(dataset_data.get("changed_on", now)).timestamp() * 1000 ) last_modified = AuditStampClass(time=modified_ts, actor=modified_actor) From 1d89fc859d94ebd14676e202c2da26aa69abc4f4 Mon Sep 17 00:00:00 2001 From: Shuixi Li Date: Mon, 17 Mar 2025 18:07:29 -0400 Subject: [PATCH 2/2] use changed_on_utc column --- .../src/datahub/ingestion/source/superset.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/superset.py b/metadata-ingestion/src/datahub/ingestion/source/superset.py index ceb5b74ce56126..bdf10bc4d308fc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/superset.py +++ b/metadata-ingestion/src/datahub/ingestion/source/superset.py @@ -401,11 +401,9 @@ def construct_dashboard_from_api_data( ) modified_actor = f"urn:li:corpuser:{self.owner_info.get((dashboard_data.get('changed_by') or {}).get('id', -1), 'unknown')}" - now = datetime.now().strftime("%I:%M%p on %B %d, %Y") - modified_ts = int( - dp.parse(dashboard_data.get("changed_on", now)).timestamp() * 1000 + dp.parse(dashboard_data.get("changed_on_utc", now)).timestamp() * 1000 ) title = dashboard_data.get("dashboard_title", "") # note: the API does not currently supply created_by usernames due to a bug @@ -517,9 +515,9 @@ def construct_chart_from_chart_data(self, chart_data: dict) -> ChartSnapshot: ) modified_actor = f"urn:li:corpuser:{self.owner_info.get((chart_data.get('changed_by') or {}).get('id', -1), 'unknown')}" - + now = datetime.now().strftime("%I:%M%p on %B %d, %Y") modified_ts = int( - dp.parse(datetime.now().strftime("%I:%M%p on %B %d, %Y")).timestamp() * 1000 + dp.parse(chart_data.get("changed_on_utc", now)).timestamp() * 1000 ) title = chart_data.get("slice_name", "") @@ -786,11 +784,9 @@ def construct_dataset_from_dataset_data( dataset_url = f"{self.config.display_uri}{dataset_response.get('result', {}).get('url', '')}" modified_actor = f"urn:li:corpuser:{self.owner_info.get((dataset_data.get('changed_by') or {}).get('id', -1), 'unknown')}" - now = datetime.now().strftime("%I:%M%p on %B %d, %Y") - modified_ts = int( - dp.parse(dataset_data.get("changed_on", now)).timestamp() * 1000 + dp.parse(dataset_data.get("changed_on_utc", now)).timestamp() * 1000 ) last_modified = AuditStampClass(time=modified_ts, actor=modified_actor)