Fix word boundary detection.

jgm · Jul 29, 2020 · 7f65eb7 · 7f65eb7
1 parent 2aaa6b0
commit 7f65eb7
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 8 deletions.
diff --git a/skylighting-core/src/Regex/KDE.hs b/skylighting-core/src/Regex/KDE.hs
@@ -1,6 +1,6 @@
 {-# LANGUAGE OverloadedStrings #-}
 module Regex.KDE
- (Regex(..), compileRegex, matchRegex, testRegex)
+ (Regex(..), compileRegex, matchRegex, testRegex, isWordChar)
   where
 
 import Regex.KDE.Regex

diff --git a/skylighting-core/src/Skylighting/Regex.hs b/skylighting-core/src/Skylighting/Regex.hs
@@ -10,6 +10,7 @@ module Skylighting.Regex (
               , compileRegex
               , matchRegex
               , testRegex
+              , isWordChar
               ) where
 
 import Data.Aeson

diff --git a/skylighting-core/src/Skylighting/Tokenizer.hs b/skylighting-core/src/Skylighting/Tokenizer.hs
@@ -567,14 +567,8 @@ wordBoundary inp = do
          c <- gets prevChar
          guard $ isWordBoundary c d
 
--- TODO is this right?
 isWordBoundary :: Char -> Char -> Bool
-isWordBoundary c d =
-  (isAlphaNum c && not (isAlphaNum d))
-  || (isAlphaNum d && not (isAlphaNum c))
-  || (isSpace d && not (isSpace c))
-  || (isSpace c && not (isSpace d))
-
+isWordBoundary c d = isWordChar c /= isWordChar d
 
 decodeBS :: ByteString -> TokenizerM Text
 decodeBS bs = case decodeUtf8' bs of