Skip to content

Commit

Permalink
✨Implemented remove
Browse files Browse the repository at this point in the history
  • Loading branch information
carefree0910 committed Oct 22, 2024
1 parent a1e62fb commit 1040e1c
Showing 1 changed file with 15 additions and 2 deletions.
17 changes: 15 additions & 2 deletions core/learn/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,11 +263,21 @@ class AsyncIterManager:

@classmethod
def new(cls, fn: Callable[[], "AsyncDataLoaderIter"]) -> "AsyncDataLoaderIter":
cls.cleanup()
cls._cur = fn()
return cls._cur

@classmethod
def remove(cls, iter: "AsyncDataLoaderIter") -> None:
if cls._cur is iter:
cls.cleanup()

@classmethod
def cleanup(cls) -> None:
if cls._cur is not None:
if not cls._cur._finalized:
cls._cur._cleanup()
cls._cur = fn()
return cls._cur
cls._cur = None


class AsyncDataLoaderIter(_SingleProcessDataLoaderIter):
Expand All @@ -290,6 +300,9 @@ def __init__(self, loader: "DataLoader"):
self._initialized = False
self._pool = ThreadPoolExecutor(max_workers=self.async_prefetch_factor)

def __del__(self) -> None:
AsyncIterManager.remove(self)

def _initialize(self) -> None:
self._queue = None
self._drained = False
Expand Down

0 comments on commit 1040e1c

Please sign in to comment.