From d91a7d43d716b5ce35c5cd34090164386c0518fe Mon Sep 17 00:00:00 2001 From: Sameera Wijerathne Date: Fri, 27 Dec 2024 17:10:28 +0400 Subject: [PATCH 01/17] Update Python connector version to 30 --- python/delta_sharing/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/delta_sharing/version.py b/python/delta_sharing/version.py index f5d25472e..8c08864d6 100644 --- a/python/delta_sharing/version.py +++ b/python/delta_sharing/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "1.1.0" +__version__ = "30" From f8b65bd26538cf2afaf50c043d2b0c36611dad21 Mon Sep 17 00:00:00 2001 From: Sameera Wijerathne Date: Fri, 27 Dec 2024 17:23:32 +0400 Subject: [PATCH 02/17] DeltaSharingProfile and Protocol version upgraded Unwrapped delta action in the response action when responseFormat=delta On branch sameerawi/rest-client-delta-table-version-upgrade Changes to be committed: modified: python/delta_sharing/protocol.py modified: python/delta_sharing/rest_client.py --- python/delta_sharing/protocol.py | 9 +++++---- python/delta_sharing/rest_client.py | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/python/delta_sharing/protocol.py b/python/delta_sharing/protocol.py index a99acfb14..069d7c115 100644 --- a/python/delta_sharing/protocol.py +++ b/python/delta_sharing/protocol.py @@ -23,7 +23,7 @@ @dataclass(frozen=True) class DeltaSharingProfile: - CURRENT: ClassVar[int] = 2 + CURRENT: ClassVar[int] = 3 share_credentials_version: int endpoint: str @@ -159,7 +159,7 @@ def from_json(json) -> "Table": @dataclass(frozen=True) class Protocol: - CURRENT: ClassVar[int] = 1 + CURRENT: ClassVar[int] = 3 min_reader_version: int @@ -175,7 +175,8 @@ def __post_init__(self): def from_json(json) -> "Protocol": if isinstance(json, (str, bytes, bytearray)): json = loads(json) - return Protocol(min_reader_version=int(json["minReaderVersion"])) + deltaProtocol = json["deltaProtocol"] + return Protocol(min_reader_version=int(deltaProtocol["minReaderVersion"])) @dataclass(frozen=True) @@ -206,7 +207,7 @@ class Metadata: @staticmethod def from_json(json) -> "Metadata": if isinstance(json, (str, bytes, bytearray)): - json = loads(json) + json = loads(json)["deltaMetadata"] if "configuration" in json: configuration = json["configuration"] else: diff --git a/python/delta_sharing/rest_client.py b/python/delta_sharing/rest_client.py index b9638f8fc..d1a24487c 100644 --- a/python/delta_sharing/rest_client.py +++ b/python/delta_sharing/rest_client.py @@ -284,8 +284,8 @@ def query_table_metadata(self, table: Table) -> QueryTableMetadataResponse: return QueryTableMetadataResponse( delta_table_version=int(headers.get( DataSharingRestClient.DELTA_TABLE_VERSION_HEADER)), - protocol=Protocol.from_json(protocol_json["protocol"]), - metadata=Metadata.from_json(metadata_json["metaData"]), + protocol=Protocol.from_json(json.dumps(protocol_json["protocol"])), + metadata=Metadata.from_json(json.dumps(metadata_json["metaData"])), ) @retry_with_exponential_backoff From bf2ff95691fdb4b43d870b7fe71ef1278f827558 Mon Sep 17 00:00:00 2001 From: Sameera Wijerathne Date: Mon, 30 Dec 2024 06:12:22 +0400 Subject: [PATCH 03/17] reverted the version back to 1.1.0 Signed-off-by: Sameera Wijerathne --- python/delta_sharing/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/delta_sharing/version.py b/python/delta_sharing/version.py index 8c08864d6..f5d25472e 100644 --- a/python/delta_sharing/version.py +++ b/python/delta_sharing/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "30" +__version__ = "1.1.0" From 9e9798f6371a3a932d01910e4f47d987be6dde3d Mon Sep 17 00:00:00 2001 From: Sameera Wijerathne Date: Mon, 30 Dec 2024 15:19:12 +0400 Subject: [PATCH 04/17] Update Python connector version to 32 --- python/delta_sharing/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/delta_sharing/version.py b/python/delta_sharing/version.py index f5d25472e..991b2e397 100644 --- a/python/delta_sharing/version.py +++ b/python/delta_sharing/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "1.1.0" +__version__ = "32" From be5f0a7ca717bf553c7c5282568eebccc4024ba6 Mon Sep 17 00:00:00 2001 From: Sameera Wijerathne Date: Mon, 30 Dec 2024 15:21:57 +0400 Subject: [PATCH 05/17] Update Python connector version to 33 --- python/delta_sharing/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/delta_sharing/version.py b/python/delta_sharing/version.py index 991b2e397..da7e382a1 100644 --- a/python/delta_sharing/version.py +++ b/python/delta_sharing/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "32" +__version__ = "33" From 7a39d2e4871d415dc830310f30e7d871dd19b16d Mon Sep 17 00:00:00 2001 From: Sameera Wijerathne Date: Mon, 30 Dec 2024 15:22:25 +0400 Subject: [PATCH 06/17] Update Python connector version to 34 --- python/delta_sharing/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/delta_sharing/version.py b/python/delta_sharing/version.py index da7e382a1..bdac812fe 100644 --- a/python/delta_sharing/version.py +++ b/python/delta_sharing/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "33" +__version__ = "34" From e50579c52c0e347ccf5c8829e19280a33f5ba0fc Mon Sep 17 00:00:00 2001 From: Sameera Wijerathne Date: Mon, 30 Dec 2024 15:22:52 +0400 Subject: [PATCH 07/17] Update Python connector version to 35 --- python/delta_sharing/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/delta_sharing/version.py b/python/delta_sharing/version.py index bdac812fe..851fab37e 100644 --- a/python/delta_sharing/version.py +++ b/python/delta_sharing/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "34" +__version__ = "35" From 5f45a22f6f810a616aacaac7c9fba7a191b68571 Mon Sep 17 00:00:00 2001 From: Sameera Wijerathne Date: Mon, 30 Dec 2024 16:03:58 +0400 Subject: [PATCH 08/17] Update Python connector version to 36 --- python/delta_sharing/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/delta_sharing/version.py b/python/delta_sharing/version.py index 851fab37e..49d4ddc4a 100644 --- a/python/delta_sharing/version.py +++ b/python/delta_sharing/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "35" +__version__ = "36" From 66d9a6e5d2d4405a546301e15b9d49d2b49339fa Mon Sep 17 00:00:00 2001 From: Sameera Wijerathne Date: Mon, 30 Dec 2024 16:10:40 +0400 Subject: [PATCH 09/17] Update Python connector version to 37 --- python/delta_sharing/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/delta_sharing/version.py b/python/delta_sharing/version.py index 49d4ddc4a..35e26486f 100644 --- a/python/delta_sharing/version.py +++ b/python/delta_sharing/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "36" +__version__ = "37" From f82dbab085836cabbde4ee0dcccd8a8037d06b27 Mon Sep 17 00:00:00 2001 From: Sameera Wijerathne Date: Mon, 30 Dec 2024 16:13:02 +0400 Subject: [PATCH 10/17] Included a condition to check if deltaMetadata and deltaProtocol wrappers are included in the response to be compatible with previous version of deltashare tables. Signed-off-by: Sameera Wijerathne --- python/delta_sharing/protocol.py | 5 ++--- python/delta_sharing/rest_client.py | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/python/delta_sharing/protocol.py b/python/delta_sharing/protocol.py index 069d7c115..289f08dfc 100644 --- a/python/delta_sharing/protocol.py +++ b/python/delta_sharing/protocol.py @@ -175,8 +175,7 @@ def __post_init__(self): def from_json(json) -> "Protocol": if isinstance(json, (str, bytes, bytearray)): json = loads(json) - deltaProtocol = json["deltaProtocol"] - return Protocol(min_reader_version=int(deltaProtocol["minReaderVersion"])) + return Protocol(min_reader_version=int(json["minReaderVersion"])) @dataclass(frozen=True) @@ -207,7 +206,7 @@ class Metadata: @staticmethod def from_json(json) -> "Metadata": if isinstance(json, (str, bytes, bytearray)): - json = loads(json)["deltaMetadata"] + json = loads(json) if "configuration" in json: configuration = json["configuration"] else: diff --git a/python/delta_sharing/rest_client.py b/python/delta_sharing/rest_client.py index d1a24487c..47d66d167 100644 --- a/python/delta_sharing/rest_client.py +++ b/python/delta_sharing/rest_client.py @@ -281,12 +281,20 @@ def query_table_metadata(self, table: Table) -> QueryTableMetadataResponse: lines = values[1] protocol_json = json.loads(next(lines)) metadata_json = json.loads(next(lines)) + + if "deltaProtocol" in protocol_json["protocol"]: + protocol = Protocol.from_json(json.dumps(protocol_json["protocol"]["deltaProtocol"])) + else: + protocol = Protocol.from_json(json.dumps(protocol_json["protocol"])) + + if "deltaMetadata" in metadata_json["metaData"]: + metadata = Metadata.from_json(json.dumps(metadata_json["metaData"]["deltaMetadata"])) + else: + metadata = Metadata.from_json(json.dumps(metadata_json["metaData"])) + return QueryTableMetadataResponse( delta_table_version=int(headers.get( - DataSharingRestClient.DELTA_TABLE_VERSION_HEADER)), - protocol=Protocol.from_json(json.dumps(protocol_json["protocol"])), - metadata=Metadata.from_json(json.dumps(metadata_json["metaData"])), - ) + DataSharingRestClient.DELTA_TABLE_VERSION_HEADER)), protocol=protocol, metadata=metadata) @retry_with_exponential_backoff def autoresolve_query_format(self, table: Table): From fa51c0a0f6641b43a0a7c2995bf482b15eb24f2d Mon Sep 17 00:00:00 2001 From: Sameera Wijerathne Date: Mon, 30 Dec 2024 16:19:12 +0400 Subject: [PATCH 11/17] Reverted the version --- python/delta_sharing/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/delta_sharing/version.py b/python/delta_sharing/version.py index 35e26486f..f5d25472e 100644 --- a/python/delta_sharing/version.py +++ b/python/delta_sharing/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "37" +__version__ = "1.1.0" From 61619d1b6b372e30a84a6659b35033424f92d45d Mon Sep 17 00:00:00 2001 From: Sameera Wijerathne Date: Mon, 30 Dec 2024 17:02:13 +0400 Subject: [PATCH 12/17] fixed the long line issue --- python/delta_sharing/rest_client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/delta_sharing/rest_client.py b/python/delta_sharing/rest_client.py index 47d66d167..26cb5c6a5 100644 --- a/python/delta_sharing/rest_client.py +++ b/python/delta_sharing/rest_client.py @@ -283,12 +283,14 @@ def query_table_metadata(self, table: Table) -> QueryTableMetadataResponse: metadata_json = json.loads(next(lines)) if "deltaProtocol" in protocol_json["protocol"]: - protocol = Protocol.from_json(json.dumps(protocol_json["protocol"]["deltaProtocol"])) + protocol = Protocol.from_json( + json.dumps(protocol_json["protocol"]["deltaProtocol"])) else: protocol = Protocol.from_json(json.dumps(protocol_json["protocol"])) if "deltaMetadata" in metadata_json["metaData"]: - metadata = Metadata.from_json(json.dumps(metadata_json["metaData"]["deltaMetadata"])) + metadata = Metadata.from_json( + json.dumps(metadata_json["metaData"]["deltaMetadata"])) else: metadata = Metadata.from_json(json.dumps(metadata_json["metaData"])) From 2914a22ed1caee01075ae7077248228c3d357234 Mon Sep 17 00:00:00 2001 From: Sameera Wijerathne Date: Mon, 30 Dec 2024 17:10:33 +0400 Subject: [PATCH 13/17] fixed the long line issue --- python/delta_sharing/rest_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/delta_sharing/rest_client.py b/python/delta_sharing/rest_client.py index 26cb5c6a5..696d6ad7a 100644 --- a/python/delta_sharing/rest_client.py +++ b/python/delta_sharing/rest_client.py @@ -296,7 +296,8 @@ def query_table_metadata(self, table: Table) -> QueryTableMetadataResponse: return QueryTableMetadataResponse( delta_table_version=int(headers.get( - DataSharingRestClient.DELTA_TABLE_VERSION_HEADER)), protocol=protocol, metadata=metadata) + DataSharingRestClient.DELTA_TABLE_VERSION_HEADER)), + protocol=protocol, metadata=metadata) @retry_with_exponential_backoff def autoresolve_query_format(self, table: Table): From d40a44d69493adb6bfb02a7232d29dca3d7babbd Mon Sep 17 00:00:00 2001 From: Sameera Wijerathne Date: Mon, 30 Dec 2024 17:18:08 +0400 Subject: [PATCH 14/17] Update Python connector version to 38 --- python/delta_sharing/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/delta_sharing/version.py b/python/delta_sharing/version.py index f5d25472e..d9db19005 100644 --- a/python/delta_sharing/version.py +++ b/python/delta_sharing/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "1.1.0" +__version__ = "38" From 3153f9a8866f5f818d7e2fb07fbf669a6605fb18 Mon Sep 17 00:00:00 2001 From: Sameera Wijerathne Date: Mon, 30 Dec 2024 17:19:02 +0400 Subject: [PATCH 15/17] Update Python connector version to 40 --- python/delta_sharing/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/delta_sharing/version.py b/python/delta_sharing/version.py index d9db19005..661b13e76 100644 --- a/python/delta_sharing/version.py +++ b/python/delta_sharing/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "38" +__version__ = "40" From a007ac1777a57c293332cd5113096023803ebda8 Mon Sep 17 00:00:00 2001 From: Sameera Wijerathne Date: Mon, 30 Dec 2024 17:29:50 +0400 Subject: [PATCH 16/17] pycodestyle indent issue fixed --- python/delta_sharing/rest_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/delta_sharing/rest_client.py b/python/delta_sharing/rest_client.py index 696d6ad7a..d5a08e0bb 100644 --- a/python/delta_sharing/rest_client.py +++ b/python/delta_sharing/rest_client.py @@ -297,7 +297,9 @@ def query_table_metadata(self, table: Table) -> QueryTableMetadataResponse: return QueryTableMetadataResponse( delta_table_version=int(headers.get( DataSharingRestClient.DELTA_TABLE_VERSION_HEADER)), - protocol=protocol, metadata=metadata) + protocol=protocol, + metadata=metadata + ) @retry_with_exponential_backoff def autoresolve_query_format(self, table: Table): From 359b2dc80d3104e9df06888758869aff08301ea1 Mon Sep 17 00:00:00 2001 From: Sameera Wijerathne Date: Mon, 30 Dec 2024 17:41:20 +0400 Subject: [PATCH 17/17] client version reverted --- python/delta_sharing/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/delta_sharing/version.py b/python/delta_sharing/version.py index 661b13e76..f5d25472e 100644 --- a/python/delta_sharing/version.py +++ b/python/delta_sharing/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "40" +__version__ = "1.1.0"