Validate the computed chunk_overlap (a float ≥ 1.0 silently drops all content)

src/chonkie/chunker/token.py

@@ -56,6 +56,13 @@ def __init__(
         self.chunk_overlap = (
             chunk_overlap if isinstance(chunk_overlap, int) else int(chunk_overlap * chunk_size)
         )
+        # Validate the COMPUTED overlap, not just the int input: a float chunk_overlap >= 1.0
+        # is turned into int(chunk_overlap * chunk_size), which can be >= chunk_size and was
+        # skipped by the int-only guard above. That makes the step (chunk_size - chunk_overlap)
+        # zero or negative, so range() yields nothing and chunk() silently returns [] - the
+        # whole document is dropped. Enforce the docstring's contract for ints and floats alike.
+        if self.chunk_overlap >= self.chunk_size or self.chunk_overlap < 0:
+            raise ValueError("chunk_overlap must be >= 0 and less than chunk_size")
 
         self._use_multiprocessing = False
 

tests/test_token_overlap_validation.py

@@ -0,0 +1,25 @@
+"""Regression test: a float chunk_overlap >= 1.0 must be rejected, not silently drop content.
+
+TokenChunker.__init__ guarded `chunk_overlap >= chunk_size` only for int inputs. A float
+like 1.5 is turned into int(1.5 * chunk_size), which can exceed chunk_size, making the step
+(chunk_size - chunk_overlap) negative -> range() is empty -> chunk() returns [] and the whole
+document is silently dropped from a RAG index. The docstring promises ValueError when
+chunk_overlap >= chunk_size; this enforces it for floats too.
+
+  with the fix -> PASS (raises ValueError at construction; valid fractional overlap still works)
+  without it   -> FAIL (no raise; chunk() silently returns [])
+"""
+import pytest
+
+from chonkie import TokenChunker
+
+
+def test_float_overlap_ge_one_is_rejected():
+    with pytest.raises(ValueError):
+        TokenChunker(tokenizer="character", chunk_size=10, chunk_overlap=1.5)
+
+
+def test_valid_fractional_overlap_still_chunks():
+    chunker = TokenChunker(tokenizer="character", chunk_size=10, chunk_overlap=0.1)  # -> 1 token
+    chunks = chunker.chunk("a" * 100)
+    assert len(chunks) > 0