diff --git a/src/chonkie/chunker/token.py b/src/chonkie/chunker/token.py index 4b8998261..5bb273da7 100644 --- a/src/chonkie/chunker/token.py +++ b/src/chonkie/chunker/token.py @@ -56,6 +56,13 @@ def __init__( self.chunk_overlap = ( chunk_overlap if isinstance(chunk_overlap, int) else int(chunk_overlap * chunk_size) ) + # Validate the COMPUTED overlap, not just the int input: a float chunk_overlap >= 1.0 + # is turned into int(chunk_overlap * chunk_size), which can be >= chunk_size and was + # skipped by the int-only guard above. That makes the step (chunk_size - chunk_overlap) + # zero or negative, so range() yields nothing and chunk() silently returns [] - the + # whole document is dropped. Enforce the docstring's contract for ints and floats alike. + if self.chunk_overlap >= self.chunk_size or self.chunk_overlap < 0: + raise ValueError("chunk_overlap must be >= 0 and less than chunk_size") self._use_multiprocessing = False diff --git a/tests/test_token_overlap_validation.py b/tests/test_token_overlap_validation.py new file mode 100644 index 000000000..88082b150 --- /dev/null +++ b/tests/test_token_overlap_validation.py @@ -0,0 +1,25 @@ +"""Regression test: a float chunk_overlap >= 1.0 must be rejected, not silently drop content. + +TokenChunker.__init__ guarded `chunk_overlap >= chunk_size` only for int inputs. A float +like 1.5 is turned into int(1.5 * chunk_size), which can exceed chunk_size, making the step +(chunk_size - chunk_overlap) negative -> range() is empty -> chunk() returns [] and the whole +document is silently dropped from a RAG index. The docstring promises ValueError when +chunk_overlap >= chunk_size; this enforces it for floats too. + + with the fix -> PASS (raises ValueError at construction; valid fractional overlap still works) + without it -> FAIL (no raise; chunk() silently returns []) +""" +import pytest + +from chonkie import TokenChunker + + +def test_float_overlap_ge_one_is_rejected(): + with pytest.raises(ValueError): + TokenChunker(tokenizer="character", chunk_size=10, chunk_overlap=1.5) + + +def test_valid_fractional_overlap_still_chunks(): + chunker = TokenChunker(tokenizer="character", chunk_size=10, chunk_overlap=0.1) # -> 1 token + chunks = chunker.chunk("a" * 100) + assert len(chunks) > 0