[nbhatt] Fix JSON parsing to handle multi-line AddFile objects in delta-sharing response

python/delta_sharing/rest_client.py

@@ -387,14 +387,36 @@ def list_files_in_table(
                     lines=[line for line in lines],
                 )
             else:
-                protocol_json = json.loads(next(lines))
-                metadata_json = json.loads(next(lines))
+                def parse_json_stream(lines):
+                    import json
+
+                    buffer = ''
+                    decoder = json.JSONDecoder()
+                    for line in lines:
+                        buffer += line.strip()
+
+                        while buffer:
+                            try:
+                                # Attempt to decode a JSON object from the buffer
+                                obj, idx = decoder.raw_decode(buffer)
+                                json_str = buffer[:idx]
+                                yield json_str
+                                buffer = buffer[idx:].lstrip()
+                            except json.JSONDecodeError:
+                                # Incomplete JSON data; read more lines
+                                break
+                parsed_lines = parse_json_stream(lines)
+                protocol_json = json.loads(next(parsed_lines))
+                metadata_json = json.loads(next(parsed_lines))
 
                 return ListFilesInTableResponse(
                     delta_table_version=int(headers.get("delta-table-version")),
                     protocol=Protocol.from_json(protocol_json["protocol"]),
                     metadata=Metadata.from_json(metadata_json["metaData"]),
-                    add_files=[AddFile.from_json(json.loads(file)["file"]) for file in lines],
+                    add_files=[
+                        AddFile.from_json(json.loads(file)["file"])
+                        for file in parsed_lines
+                    ],
                     lines=[]
                 )