Skip to content

Commit f3ba7c9

Browse files
committed
refactor: Address code review comments
1 parent 7ac3d44 commit f3ba7c9

File tree

2 files changed

+19
-15
lines changed

2 files changed

+19
-15
lines changed

src/together/filemanager.py

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,10 @@
66
import stat
77
import tempfile
88
import uuid
9-
from concurrent.futures import ThreadPoolExecutor, as_completed
9+
from concurrent.futures import Future, ThreadPoolExecutor, as_completed
1010
from functools import partial
1111
from pathlib import Path
12-
from typing import Any, Dict, List, Tuple
12+
from typing import Any, BinaryIO, Dict, List, Tuple
1313

1414
import requests
1515
from filelock import FileLock
@@ -513,6 +513,18 @@ def _initiate_upload(
513513

514514
return response.data
515515

516+
def _submit_part(
517+
self,
518+
executor: ThreadPoolExecutor,
519+
f: BinaryIO,
520+
part_info: Dict[str, Any],
521+
part_size: int,
522+
) -> Future[str]:
523+
"""Submit a single part for upload and return the future"""
524+
f.seek((part_info["PartNumber"] - 1) * part_size)
525+
part_data = f.read(part_size)
526+
return executor.submit(self._upload_single_part, part_info, part_data)
527+
516528
def _upload_parts_concurrent(
517529
self, file: Path, upload_info: Dict[str, Any], part_size: int
518530
) -> List[Dict[str, Any]]:
@@ -530,12 +542,7 @@ def _upload_parts_concurrent(
530542
# Submit initial batch limited by max_concurrent_parts
531543
for i in range(min(self.max_concurrent_parts, len(parts))):
532544
part_info = parts[part_index]
533-
f.seek((part_info["PartNumber"] - 1) * part_size)
534-
part_data = f.read(part_size)
535-
536-
future = executor.submit(
537-
self._upload_single_part, part_info, part_data
538-
)
545+
future = self._submit_part(executor, f, part_info, part_size)
539546
future_to_part[future] = part_info["PartNumber"]
540547
part_index += 1
541548

@@ -556,11 +563,8 @@ def _upload_parts_concurrent(
556563
# Submit next part if available
557564
if part_index < len(parts):
558565
part_info = parts[part_index]
559-
f.seek((part_info["PartNumber"] - 1) * part_size)
560-
part_data = f.read(part_size)
561-
562-
future = executor.submit(
563-
self._upload_single_part, part_info, part_data
566+
future = self._submit_part(
567+
executor, f, part_info, part_size
564568
)
565569
future_to_part[future] = part_info["PartNumber"]
566570
part_index += 1

src/together/utils/files.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -366,9 +366,9 @@ def _check_utf8(file: Path) -> Dict[str, Any]:
366366
report_dict: Dict[str, Any] = {}
367367

368368
try:
369-
chunk_size = 8192
369+
# Dry-run UTF-8 decode: iterate through file to validate encoding
370370
with file.open(encoding="utf-8") as f:
371-
for chunk in iter(lambda: f.read(chunk_size), ""):
371+
for _ in f:
372372
pass
373373

374374
report_dict["utf8"] = True

0 commit comments

Comments
 (0)