fuzzy matching operator (#73)

Author: rabbitstack

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/hillu/go-yara/v4 v4.0.6
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jedib0t/go-pretty/v6 v6.2.1
+	github.com/lithammer/fuzzysearch v1.1.2
 	github.com/magiconair/properties v1.8.1
 	github.com/mitchellh/mapstructure v1.4.1
 	github.com/olivere/elastic/v7 v7.0.20
@@ -26,7 +27,7 @@ require (
 	github.com/valyala/gozstd v1.6.4
 	github.com/xeipuuv/gojsonschema v1.2.0
 	golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b
-	golang.org/x/text v0.3.2
+	golang.org/x/text v0.3.5
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0

go.sum

@@ -97,6 +97,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lithammer/fuzzysearch v1.1.2 h1:ePUtm14xKxbpCxozcFbIDRtvANxnVnE+RKpJUqkr2gA=
+github.com/lithammer/fuzzysearch v1.1.2/go.mod h1:v6tYW/9kpfV6LNcweXdSjQsfCku/1M/oObmSox1fzP8=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
@@ -238,8 +240,9 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

pkg/filter/filter_test.go

@@ -192,6 +192,11 @@ func TestFilterRunFileKevent(t *testing.T) {
 		{`file.name iendswith ('.EXE', 'KERNEL32.dll', 'user32.dll')`, true},
 		{`file.name istartswith ('C:\\WINDOWS', 'KERNEL32.dll', 'user32.dll')`, true},
 		{`file.name iin ('C:\\WINDOWS\\system32\\user32.dll')`, true},
+		{`file.name fuzzy 'C:\\Windows\\system32\\ser3ll'`, true},
+		{`file.name ifuzzy 'C:\\WINDOWS\\sYS\\ser3ll'`, true},
+		{`file.name ifuzzy 'C:\\WINDOWS\\sYS\\32dll'`, true},
+		{`file.name fuzzy ('C:\\Windows\\system32\\kernel', 'C:\\Windows\\system32\\ser3ll')`, true},
+		{`file.name ifuzzynorm 'C:\\WINDOWS\\sÝS\\32dll'`, true},
 	}
 
 	for i, tt := range tests {

pkg/filter/ql/ast.go

@@ -21,6 +21,7 @@
 package ql
 
 import (
+	fuzzysearch "github.com/lithammer/fuzzysearch/fuzzy"
 	"github.com/rabbitstack/fibratus/pkg/util/wildcard"
 	"net"
 	"strconv"
@@ -803,6 +804,62 @@ func (v *ValuerEval) evalBinaryExpr(expr *BinaryExpr) interface{} {
 			default:
 				return false
 			}
+		case fuzzy:
+			switch rhs := rhs.(type) {
+			case string:
+				return fuzzysearch.Match(rhs, lhs)
+			case []string:
+				for _, s := range rhs {
+					if fuzzysearch.Match(s, lhs) {
+						return true
+					}
+				}
+				return false
+			default:
+				return false
+			}
+		case ifuzzy:
+			switch rhs := rhs.(type) {
+			case string:
+				return fuzzysearch.MatchFold(rhs, lhs)
+			case []string:
+				for _, s := range rhs {
+					if fuzzysearch.MatchFold(s, lhs) {
+						return true
+					}
+				}
+				return false
+			default:
+				return false
+			}
+		case fuzzynorm:
+			switch rhs := rhs.(type) {
+			case string:
+				return fuzzysearch.MatchNormalized(rhs, lhs)
+			case []string:
+				for _, s := range rhs {
+					if fuzzysearch.MatchNormalized(s, lhs) {
+						return true
+					}
+				}
+				return false
+			default:
+				return false
+			}
+		case ifuzzynorm:
+			switch rhs := rhs.(type) {
+			case string:
+				return fuzzysearch.MatchNormalizedFold(rhs, lhs)
+			case []string:
+				for _, s := range rhs {
+					if fuzzysearch.MatchNormalizedFold(s, lhs) {
+						return true
+					}
+				}
+				return false
+			default:
+				return false
+			}
 		}
 	case net.IP:
 		switch expr.Op {

pkg/filter/ql/token.go

@@ -52,12 +52,16 @@ const (
 	not         // not
 	contains    // contains
 	icontains   // icontains
-	startswith  // startswith
 	istartswith // istartswith
-	iendswith   // iendswith
+	startswith  // startswith
 	endswith    // endswith
+	iendswith   // iendswith
 	matches     // matches
 	imatches    // imatches
+	fuzzy       // fuzzy
+	ifuzzy      // ifuzzy
+	fuzzynorm   // fuzzynorm
+	ifuzzynorm  // ifuzzynorm
 	eq          // =
 	neq         // !=
 	lt          // <
@@ -76,7 +80,9 @@ var keywords map[string]token
 
 func init() {
 	keywords = make(map[string]token)
-	for _, tok := range []token{and, or, contains, icontains, in, iin, not, startswith, istartswith, endswith, iendswith, matches, imatches} {
+	for _, tok := range []token{and, or, contains, icontains, in,
+		iin, not, startswith, istartswith, endswith, iendswith,
+		matches, imatches, fuzzy, ifuzzy, fuzzynorm, ifuzzynorm} {
 		keywords[strings.ToLower(tokens[tok])] = tok
 	}
 	keywords["true"] = truet
@@ -114,6 +120,10 @@ var tokens = [...]string{
 	iendswith:   "IENDSWITH",
 	matches:     "MATCHES",
 	imatches:    "IMATCHES",
+	fuzzy:       "FUZZY",
+	ifuzzy:      "IFUZZY",
+	fuzzynorm:   "FUZZYNORM",
+	ifuzzynorm:  "IFUZZYNORM",
 
 	eq:  "=",
 	neq: "!=",
@@ -150,8 +160,8 @@ func (tok token) precedence() int {
 		return 3
 	case eq, neq, lt, lte, gt, gte:
 		return 4
-	case in, iin, contains, icontains, startswith, istartswith,
-		endswith, iendswith, matches, imatches:
+	case in, iin, contains, icontains, startswith, istartswith, endswith, iendswith,
+		matches, imatches, fuzzy, ifuzzy, fuzzynorm, ifuzzynorm:
 		return 5
 	}
 	return 0