Skip to content

Commit 2a3d3fa

Browse files
refactor: Use context manager to read gzip batch files (#2628)
* refactor: Use context manager to read gzip batch files * Remove redundant variable
1 parent ddb922a commit 2a3d3fa

File tree

1 file changed

+11
-9
lines changed

1 file changed

+11
-9
lines changed

singer_sdk/sinks/core.py

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -611,7 +611,7 @@ def _after_process_record(self, context: dict) -> None:
611611

612612
# SDK developer overrides:
613613

614-
def preprocess_record(self, record: dict, context: dict) -> dict: # noqa: ARG002, PLR6301
614+
def preprocess_record(self, record: dict, context: dict) -> dict: # noqa: PLR6301, ARG002
615615
"""Process incoming record and return a modified result.
616616
617617
Args:
@@ -743,12 +743,15 @@ def process_batch_files(
743743
tail,
744744
mode="rb",
745745
) as file:
746-
context_file = (
747-
gzip_open(file) if encoding.compression == "gzip" else file
748-
)
749-
context = {
750-
"records": [deserialize_json(line) for line in context_file] # type: ignore[attr-defined]
751-
}
746+
if encoding.compression == "gzip":
747+
with gzip_open(file) as context_file:
748+
context = {
749+
"records": [
750+
deserialize_json(line) for line in context_file
751+
]
752+
}
753+
else:
754+
context = {"records": [deserialize_json(line) for line in file]}
752755
self.process_batch(context)
753756
elif (
754757
importlib.util.find_spec("pyarrow")
@@ -760,8 +763,7 @@ def process_batch_files(
760763
tail,
761764
mode="rb",
762765
) as file:
763-
context_file = file
764-
table = pq.read_table(context_file)
766+
table = pq.read_table(file)
765767
context = {"records": table.to_pylist()}
766768
self.process_batch(context)
767769
else:

0 commit comments

Comments
 (0)