Merge pull request #36 from Defender32/improves-parser

improves parser multi byte string

lib/Doctrine/Common/Lexer/AbstractLexer.php

@@ -300,7 +300,7 @@ public function getLiteral($token)
      */
     protected function getModifiers()
     {
-        return 'i';
+        return 'iu';
     }
 
     /**

tests/Doctrine/Common/Lexer/AbstractLexerTest.php

@@ -5,8 +5,10 @@
 namespace Doctrine\Tests\Common\Lexer;
 
 use PHPUnit\Framework\TestCase;
+use const LC_ALL;
 use function array_map;
 use function count;
+use function setlocale;
 
 class AbstractLexerTest extends TestCase
 {
@@ -18,6 +20,14 @@ public function setUp() : void
         $this->concreteLexer = new ConcreteLexer();
     }
 
+    /**
+     * {@inheritdoc}
+     */
+    public function tearDown() : void
+    {
+        setlocale(LC_ALL, null);
+    }
+
     public function dataProvider()
     {
         return [
@@ -286,4 +296,23 @@ public function testAddCatchablePatternsToMutableLexer()
 
         $this->assertEquals('one', $token['value']);
     }
+
+    public function testMarkerAnnotationLocaleTr() : void
+    {
+        setlocale(LC_ALL, 'tr_TR.utf8', 'tr_TR');
+        $mutableLexer = new MutableLexer();
+        $mutableLexer->addCatchablePattern('[a-z_\\\][a-z0-9_\:\\\]*[a-z_][a-z0-9_]*');
+        $mutableLexer->addCatchablePattern('(?:[+-]?[0-9]+(?:[\.][0-9]+)*)(?:[eE][+-]?[0-9]+)?');
+        $mutableLexer->addCatchablePattern('"(?:""|[^"])*+"');
+        $mutableLexer->setInput('@ODM\Id');
+
+        self::assertNull($mutableLexer->token);
+        self::assertNull($mutableLexer->lookahead);
+        self::assertTrue($mutableLexer->moveNext());
+        self::assertNull($mutableLexer->token);
+        self::assertEquals('@', $mutableLexer->lookahead['value']);
+        self::assertTrue($mutableLexer->moveNext());
+        self::assertEquals('@', $mutableLexer->token['value']);
+        self::assertEquals('ODM\Id', $mutableLexer->lookahead['value']);
+    }
 }

tests/Doctrine/Common/Lexer/ConcreteLexer.php

@@ -46,9 +46,4 @@ protected function getType(&$value)
 
         return null;
     }
-
-    protected function getModifiers()
-    {
-        return parent::getModifiers() . 'u';
-    }
 }