v2.0.2

* Add support for utf-8 characters when parsing words.

Author: gdbrown

.gitignore

@@ -17,6 +17,7 @@ phpunit.xml
 .buildpath
 *.iml
 .idea/
+.phpunit.result.cache
 .project
 .settings
 

CHANGELOG-2.x.md

@@ -2,6 +2,10 @@
 This changelog references the relevant changes done in 2.x versions.
 
 
+## v2.0.2
+* Add support for utf-8 characters when parsing words.
+
+
 ## v2.0.1
 * Do not truncate input in `Tokenizer::scan`. Removed `substr($input, 0, 256)` rule as we're unsure where/why it's there and seems safe to remove.
 

src/Tokenizer.php

@@ -9,8 +9,8 @@
 use Gdbots\QueryParser\Node\Field;
 use Gdbots\QueryParser\Node\Hashtag;
 use Gdbots\QueryParser\Node\Mention;
-use Gdbots\QueryParser\Node\Numbr;
 use Gdbots\QueryParser\Node\NumberRange;
+use Gdbots\QueryParser\Node\Numbr;
 use Gdbots\QueryParser\Node\Phrase;
 use Gdbots\QueryParser\Node\Subquery;
 use Gdbots\QueryParser\Node\Url;
@@ -1328,6 +1328,19 @@
             new Word('$p0rty-spicé'),
         ],
     ],
+
+    [
+        'name'            => 'utf chars',
+        'input'           => '测试 測試',
+        'expected_tokens' => [
+            [T::T_WORD, '测试'],
+            [T::T_WORD, '測試'],
+        ],
+        'expected_nodes'  => [
+            new Word('测试'),
+            new Word('測試'),
+        ],
+    ],
     /*
      * END: ACCENTED CHARS
      */