From cd61bf19909d9752807fc6af10619620fdf01183 Mon Sep 17 00:00:00 2001 From: Lucas Date: Sun, 28 Aug 2022 11:48:37 -0500 Subject: [PATCH 1/6] Updating singer_stream.py to allow schemas without properties While utilizing tap-zendesk (https://github.com/twilio-labs/twilio-tap-zendesk) there are some cases where the generated schema does not include a `properties` key in the dictionary, causing the `properties = self.schema['properties']` to fail. Given that there is nothing returned from this function, returning early appears to resolve it. --- target_postgres/singer_stream.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/target_postgres/singer_stream.py b/target_postgres/singer_stream.py index 871d1c2c..d77bfb86 100644 --- a/target_postgres/singer_stream.py +++ b/target_postgres/singer_stream.py @@ -71,7 +71,11 @@ def update_schema(self, schema, key_properties): # The validator can handle _many_ more things than our simplified schema, and is, in general handled by third party code self.validator = Draft4Validator(schema, format_checker=FormatChecker()) - + + # Some taps do not provide 'properties' key, which causes the next block to fail + if not (self.schema and 'properties' in self.schema): + return + properties = self.schema['properties'] if singer.RECEIVED_AT not in properties: From 4400e6aee3c79a1d04c76160fb456256171fc201 Mon Sep 17 00:00:00 2001 From: Lucas Date: Wed, 31 Aug 2022 22:14:26 -0500 Subject: [PATCH 2/6] Setting schema properties check to return empty dictionary if key does not exist --- target_postgres/singer_stream.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/target_postgres/singer_stream.py b/target_postgres/singer_stream.py index d77bfb86..f94e17a1 100644 --- a/target_postgres/singer_stream.py +++ b/target_postgres/singer_stream.py @@ -72,11 +72,7 @@ def update_schema(self, schema, key_properties): # The validator can handle _many_ more things than our simplified schema, and is, in general handled by third party code self.validator = Draft4Validator(schema, format_checker=FormatChecker()) - # Some taps do not provide 'properties' key, which causes the next block to fail - if not (self.schema and 'properties' in self.schema): - return - - properties = self.schema['properties'] + properties = self.schema.get('properties', {}) if singer.RECEIVED_AT not in properties: properties[singer.RECEIVED_AT] = { From 9bf7d2a6b4db0535c4a585df796097ce3c077558 Mon Sep 17 00:00:00 2001 From: Lucas Date: Wed, 31 Aug 2022 22:15:31 -0500 Subject: [PATCH 3/6] Whitespace cleanup --- target_postgres/singer_stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target_postgres/singer_stream.py b/target_postgres/singer_stream.py index f94e17a1..c67361cc 100644 --- a/target_postgres/singer_stream.py +++ b/target_postgres/singer_stream.py @@ -71,7 +71,7 @@ def update_schema(self, schema, key_properties): # The validator can handle _many_ more things than our simplified schema, and is, in general handled by third party code self.validator = Draft4Validator(schema, format_checker=FormatChecker()) - + properties = self.schema.get('properties', {}) if singer.RECEIVED_AT not in properties: From 2b569b766b6854a55036be5911bf3e4dc293949b Mon Sep 17 00:00:00 2001 From: Lucas Date: Fri, 18 Nov 2022 21:23:29 -0600 Subject: [PATCH 4/6] Update setup.py Updating dependencies --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 518fda11..addbe088 100644 --- a/setup.py +++ b/setup.py @@ -19,9 +19,9 @@ classifiers=['Programming Language :: Python :: 3 :: Only'], py_modules=['target_postgres'], install_requires=[ - 'arrow==0.15.5', - 'psycopg2==2.8.5', - 'singer-python==5.9.0' + 'arrow==1.2.3', + 'psycopg2==2.9.5', + 'singer-python==5.13.0' ], setup_requires=[ "pytest-runner" From e4ddb929084c75eb0c865223d64d4c1cac313750 Mon Sep 17 00:00:00 2001 From: Lucas Date: Fri, 18 Nov 2022 22:16:28 -0600 Subject: [PATCH 5/6] Update singer_stream.py update arrow dep --- target_postgres/singer_stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target_postgres/singer_stream.py b/target_postgres/singer_stream.py index c67361cc..c13e980f 100644 --- a/target_postgres/singer_stream.py +++ b/target_postgres/singer_stream.py @@ -182,7 +182,7 @@ def get_batch(self): if 'sequence' in record_message: record[singer.SEQUENCE] = record_message['sequence'] else: - record[singer.SEQUENCE] = arrow.get().timestamp + record[singer.SEQUENCE] = arrow.get().timestamp() records.append(record) From b1f948fed399c9c47cd4d80e1b892ce9a52659e6 Mon Sep 17 00:00:00 2001 From: Lucas Date: Fri, 18 Nov 2022 22:18:53 -0600 Subject: [PATCH 6/6] Update singer_stream.py fix type --- target_postgres/singer_stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target_postgres/singer_stream.py b/target_postgres/singer_stream.py index c13e980f..ba2b5fc3 100644 --- a/target_postgres/singer_stream.py +++ b/target_postgres/singer_stream.py @@ -182,7 +182,7 @@ def get_batch(self): if 'sequence' in record_message: record[singer.SEQUENCE] = record_message['sequence'] else: - record[singer.SEQUENCE] = arrow.get().timestamp() + record[singer.SEQUENCE] = int(arrow.get().timestamp()) records.append(record)