Customize logical operators by subclassing Lexer. (#40)

Author: jg-rp

docs/advanced.md

@@ -193,7 +193,26 @@ This table shows all available identifier token attributes.
 
 ### Logical Operator Tokens
 
-TODO:
+By default, we accept both Python and C-style logical operators in filter expressions. That is, `not` and `!` are equivalent, `and` and `&&` are equivalent and `or` and `||` are equivalent. You can change this using class attributes on a [`Lexer`](custom_api.md#jsonpath.lex.Lexer) subclass and setting the `lexer_class` attribute on a `JSONPathEnvironment`.
+
+This example changes all three logical operators to strictly match the JSONPath spec.
+
+```python
+from jsonpath import JSONPathEnvironment
+from jsonpath import Lexer
+
+class MyLexer(Lexer):
+    logical_not_pattern = r"!"
+    logical_and_pattern = r"&&"
+    logical_or_pattern = r"\|\|"
+
+class MyJSONPathEnvironment(JSONPathEnvironment):
+    lexer_class = MyLexer
+
+env = MyJSONPathEnvironment()
+env.compile("$.foo[[email protected] > 0 && @.b < 100]")  # OK
+env.compile("$.foo[[email protected] > 0 and @.b < 100]")  # JSONPathSyntaxError
+```
 
 ### Keys Selector
 

docs/custom_api.md

@@ -1,14 +1,15 @@
 # Low Level API Reference
 
 ::: jsonpath.token.Token
+
     handler: python
 
 ::: jsonpath.filter.FilterExpression
-    handler: python
 
-## jsonpath.lex.Lexer
+    handler: python
 
-TODO:
+::: jsonpath.lex.Lexer
+    handler: python
 
 ## jsonpath.parse.Parser
 

jsonpath/lex.py

@@ -66,10 +66,34 @@
 
 
 class Lexer:
-    """Tokenize a JSONPath string."""
+    """Tokenize a JSONPath string.
+
+    Some customization can be achieved by subclassing _Lexer_ and setting
+    class attributes. Then setting `lexer_class` on a `JSONPathEnvironment`.
+
+    Attributes:
+        key_pattern: The regular expression pattern used to match mapping
+            keys/properties.
+        logical_not_pattern: The regular expression pattern used to match
+            logical negation tokens. By default, `not` and `!` are
+            equivalent.
+        logical_and_pattern: The regular expression pattern used to match
+            logical _and_ tokens. By default, `and` and `&&` are equivalent.
+        logical_or_pattern: The regular expression pattern used to match
+            logical _or_ tokens. By default, `or` and `||` are equivalent.
+    """
 
     key_pattern = r"[\u0080-\uFFFFa-zA-Z_][\u0080-\uFFFFa-zA-Z0-9_-]*"
 
+    # `not` or !
+    logical_not_pattern = r"(?:not|!)"
+
+    # && or `and`
+    logical_and_pattern = r"(?:&&|and)"
+
+    # || or `or`
+    logical_or_pattern = r"(?:\|\||or)"
+
     def __init__(self, *, env: JSONPathEnvironment) -> None:
         self.env = env
 
@@ -85,15 +109,6 @@ def __init__(self, *, env: JSONPathEnvironment) -> None:
             r"(?::\s*(?P<G_LSLICE_STEP>\-?\d*))?"
         )
 
-        # `not` or !
-        self.logical_not_pattern = r"(?:not|!)"
-
-        # && or `and`
-        self.bool_and_pattern = r"(?:&&|and)"
-
-        # || or `or`
-        self.bool_or_pattern = r"(?:\|\||or)"
-
         # /pattern/ or /pattern/flags
         self.re_pattern = r"/(?P<G_RE>.+?)/(?P<G_RE_FLAGS>[aims]*)"
 
@@ -114,8 +129,8 @@ def compile_rules(self) -> Pattern[str]:
             (TOKEN_FLOAT, r"-?\d+\.\d*(?:e[+-]?\d+)?"),
             (TOKEN_INT, r"-?\d+(?P<G_EXP>e[+\-]?\d+)?\b"),
             (TOKEN_DDOT, r"\.\."),
-            (TOKEN_AND, self.bool_and_pattern),
-            (TOKEN_OR, self.bool_or_pattern),
+            (TOKEN_AND, self.logical_and_pattern),
+            (TOKEN_OR, self.logical_or_pattern),
             (TOKEN_ROOT, re.escape(self.env.root_token)),
             (TOKEN_SELF, re.escape(self.env.self_token)),
             (TOKEN_KEY, re.escape(self.env.key_token)),

jsonpath/parse.py

@@ -446,6 +446,7 @@ def parse_selector_list(self, stream: TokenStream) -> ListSelector:  # noqa: PLR
                 )
 
             if stream.peek.kind != TOKEN_RBRACKET:
+                # TODO: error message .. expected a comma or logical operator
                 stream.expect_peek(TOKEN_COMMA)
                 stream.next_token()