corrected wrong xNext behavior

merged suggest changes for issue #1 by saaj
 * Tokenizer.tokenize returns tokenized text, start/end position in given text in bytes, and it changed a generator.
 * sets position/length in the correct unit(bytes instead of length of unicode string)
 * skips empty token

Author: hideaki-t

.gitignore

@@ -1,5 +1,7 @@
-*.py[co]
-*.sw[po]
-__pycache__
-*~
-
+.tox/
+build/
+dist/
+env/
+sqlitefts.egg-info/
+*.pyc
+*.py~

setup.py

@@ -1,7 +1,28 @@
 from setuptools import setup
+import sys
+
 
 setup(
     name="sqlitefts",
     version="0.1",
-    packages=["sqlitefts"]
+    packages=["sqlitefts"],
+    description='A Python binding of SQLite Full Text Search Tokenizer',
+    url='https://github.com/hideaki-t/igo-python/',
+    classifiers=[
+        'Development Status :: 2 - Pre-Alpha',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: MIT License',
+        'Operating System :: OS Independent',
+        'Operating System :: POSIX :: Linux',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3.3',
+        'Programming Language :: Python :: 3.4',
+        'Topic :: Database'
+        'Topic :: Software Development :: Libraries :: Python Modules'
+    ],
+    author='Hideaki Takahashi',
+    author_email='[email protected]',
+    license='MIT',
+    keywords=['SQLite', 'Full-text search', 'FTS'],
+    install_requires=['enum34'] if sys.version_info < (3, 4) else []
 )

sqlitefts/sqlite_tokenizer.py

@@ -9,6 +9,16 @@
 from ctypes import POINTER, CFUNCTYPE
 import struct
 
+try:
+    from enum import Enum
+except:
+    pass
+
+
+class SQLiteResultCodes(Enum):
+    SQLITE_OK = 0
+    SQLITE_DONE = 101
+
 
 class sqlite3_tokenizer_module(ctypes.Structure):
     pass
@@ -46,8 +56,8 @@ class sqlite3_tokenizer_cursor(ctypes.Structure):
 class Tokenizer:
     """ Tokenizer base class """
     def tokenize(text):
-        """ Tokenizer given unicode text. Returns an iterator of token """
-        return text
+        """ Tokenize given unicode text. Yields each tokenized token, start position(in bytes), end positon(in bytes)"""
+        yield text, 0, len(text.encode('utf-8'))
 
 
 tokenizer_modules = {}
@@ -64,11 +74,11 @@ def xcreate(argc, argv, ppTokenizer):
         tkn.t = tokenizer
         tokenizers[ctypes.addressof(tkn)] = tkn
         ppTokenizer[0] = ctypes.pointer(tkn)
-        return 0
+        return SQLiteResultCodes.SQLITE_OK.value
 
     def xdestroy(pTokenizer):
         del(tokenizers[ctypes.addressof(pTokenizer[0])])
-        return 0
+        return SQLiteResultCodes.SQLITE_OK.value
 
     def xopen(pTokenizer, pInput, nInput, ppCursor):
         cur = sqlite3_tokenizer_cursor()
@@ -78,28 +88,33 @@ def xopen(pTokenizer, pInput, nInput, ppCursor):
         cur.offset = 0
         cursors[ctypes.addressof(cur)] = cur
         ppCursor[0] = ctypes.pointer(cur)
-        return 0
+        return SQLiteResultCodes.SQLITE_OK.value
 
     def xnext(pCursor, ppToken, pnBytes,
               piStartOffset, piEndOffset, piPosition):
         try:
             cur = pCursor[0]
-            token = next(cur.tokens).encode('utf-8')
-            tokenlen = len(token)
-            ppToken[0] = token
-            pnBytes[0] = tokenlen
-            piStartOffset[0] = cur.offset
-            cur.offset += tokenlen
-            piEndOffset[0] = cur.offset
+
+            while True:
+                normalized, inputBegin, inputEnd = next(cur.tokens)
+                normalized = normalized.encode('utf-8')
+                if normalized:
+                    break
+
+            ppToken[0] = normalized
+            pnBytes[0] = len(normalized)
+            piStartOffset[0] = inputBegin
+            piEndOffset[0] = inputEnd
+            cur.offset = inputEnd
             piPosition[0] = cur.pos
             cur.pos += 1
         except StopIteration:
-            return 101
-        return 0
+            return SQLiteResultCodes.SQLITE_DONE.value
+        return SQLiteResultCodes.SQLITE_OK.value
 
     def xclose(pCursor):
         del(cursors[ctypes.addressof(pCursor[0])])
-        return 0
+        return SQLiteResultCodes.SQLITE_OK.value
 
     tokenizer_module = sqlite3_tokenizer_module(
         0,

tests/test_base.py

@@ -1,16 +1,21 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
-import sys
-import os
 import sqlite3
 import ctypes
 import struct
+import re
 
 import sqlitefts.sqlite_tokenizer as fts
 
+
 class SimpleTokenizer(fts.Tokenizer):
+    _p = re.compile(r'\S+')
+
     def tokenize(self, text):
-        return iter(text.split(' '))
+        for m in self._p.finditer(text):
+            s, e = m.span()
+            yield text[s:e], s, e
+
 
 def test_make_tokenizer():
     c = sqlite3.connect(':memory:')

tests/test_base2.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+from __future__ import print_function
+
+import unittest
+import sqlite3
+import re
+
+import sqlitefts.sqlite_tokenizer as fts
+
+
+class BaseTokenizer(fts.Tokenizer):
+
+    _spliter = re.compile(r'\s+|\S+')
+    _nonws = re.compile(r'\S+')
+
+    def _normalize(self, token):
+        return token
+
+    def _tokenize(self, text):
+        pos = 0
+        for t in self._spliter.findall(text):
+            byteLen = len(t.encode('utf-8'))
+            if self._nonws.match(t):
+                yield self._normalize(t), pos, pos + byteLen
+            pos += byteLen
+
+    def tokenize(self, text):
+        return self._tokenize(text)
+
+
+class DebugTokenizer(BaseTokenizer):
+
+    _limit = 16
+
+    def _normalize(self, token):
+        if not self._limit:
+            raise RuntimeError()
+        self._limit -= 1
+
+        print(token, token[0:-1])
+        return token[0:-1]
+
+
+class OriginalDebugTokenizer(fts.Tokenizer):
+
+    _limit = 16
+
+    def tokenize(self, text):
+        if not self._limit:
+            raise RuntimeError()
+        self._limit -= 1
+
+        print(text, [w[0:-1] for w in text.split(' ')])
+        return (w[0:-1] for w in text.split(' '))
+
+
+class TestCase(unittest.TestCase):
+
+    def setUp(self):
+        name = 'test'
+        conn = sqlite3.connect(':memory:')
+
+        fts.register_tokenizer(conn, name, fts.make_tokenizer_module(DebugTokenizer()))
+        conn.execute('CREATE VIRTUAL TABLE fts USING FTS4(tokenize={})'.format(name))
+
+        self.testee = conn
+
+    def testZeroLengthToken(self):
+        result = self.testee.executemany('INSERT INTO fts VALUES(?)', [('Make things I',), (u'Some σ φχικλψ',)])
+        self.assertEqual(2, result.rowcount)
+
+    def testInfiniteRecursion(self):
+        contents = [('abc def',), ('abc xyz',)]
+        result = self.testee.executemany('INSERT INTO fts VALUES(?)', contents)
+        self.assertEqual(2, result.rowcount)
+
+        result = self.testee.execute("SELECT * FROM fts WHERE fts MATCH 'abc'").fetchall()
+        self.assertEqual(2, len(result))

tests/test_igo.py

@@ -1,7 +1,5 @@
 # coding: utf-8
 from __future__ import print_function, unicode_literals
-import sys
-import os
 import sqlite3
 import ctypes
 import struct
@@ -11,16 +9,19 @@
 import pytest
 igo = pytest.importorskip('igo')
 
+
 class IgoTokenizer(fts.Tokenizer):
     def __init__(self, path=None):
         self.tagger = igo.tagger.Tagger(path)
 
     def tokenize(self, text):
-        return iter([m.surface for m in self.tagger.parse(text)])
+        for m in self.tagger.parse(text):
+            yield m.surface, m.start, m.start + len(m.surface.encode('utf-8'))
 
 
 t = IgoTokenizer('./ipadic')
 
+
 def test_make_tokenizer():
     c = sqlite3.connect(':memory:')
     tokenizer_module = fts.make_tokenizer_module(t)

tests/test_tinysegmenter.py

@@ -12,16 +12,22 @@
 import pytest
 ts = pytest.importorskip('tinysegmenter')
 
+
 class TinySegmenterTokenizer(fts.Tokenizer):
     def __init__(self, path=None):
         self.segmenter = ts.TinySegmenter()
 
     def tokenize(self, text):
-        return iter(self.segmenter.tokenize(text))
+        p = 0
+        for t in self.segmenter.tokenize(text):
+            np = text[p:].index(t)
+            yield t, np, len(t.encode('utf-8'))
+            p = np
 
 
 t = TinySegmenterTokenizer()
 
+
 def test_make_tokenizer():
     c = sqlite3.connect(':memory:')
     tokenizer_module = fts.make_tokenizer_module(t)

tox.ini

@@ -1,12 +1,29 @@
 [tox]
-envlist = py27, py33
-#envlist = py27, py33, pypy
+envlist = py27, py33, py34
 
 [testenv]
 changedir=tests
-deps = pytest
-       igo-python
-       git+git://git.tuxfamily.org/gitroot/tinysegmente/tinysegmenter.git
+deps=
+     pytest
+     igo-python
+     git+git://git.tuxfamily.org/gitroot/tinysegmente/tinysegmenter.git
 #tinysegmenter
-commands = py.test -sv
+commands=
+    py.test -sv
+
+[testenv:pp34]
+deps=
+    {[testenv]deps}
+    enum34
+
+[testenv:py33]
+deps=
+    {[testenv:pp34]deps}
+[testenv:py27]
+deps=
+    {[testenv:pp34]deps}
+
+[testenv:pypy]
+deps=
+    {[testenv:pp34]deps}