add option to ignore failed connections

config.hjson

@@ -2,6 +2,7 @@
     update_check_enabled: true,
     filtering: {
         filter_external_to_internal: true,
+        filter_failed_connections: false,
         internal_subnets: ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fd00::/8"]
     },
     threat_intel: {

config/config.go

@@ -543,6 +543,7 @@ func defaultConfig() Config {
 			AlwaysIncludedDomains:     []string{},
 			NeverIncludedDomains:      []string{},
 			FilterExternalToInternal:  true,
+			FilterFailedConnections:   false,
 		},
 		HTTPExtensionsFilePath:          "./http_extensions_list.csv",
 		BatchSize:                       100000,

config/config_test.go

@@ -54,6 +54,7 @@ func TestReadFileConfig(t *testing.T) {
 						always_included_domains: ["abc.com", "def.com"],
 						never_included_domains: ["ghi.com", "jkl.com"],
 						filter_external_to_internal: false,
+						filter_failed_connections: false,
 					},
 					http_extensions_file_path: "/path/to/http/extensions",
 					batch_size: 75000,
@@ -151,6 +152,7 @@ func TestReadFileConfig(t *testing.T) {
 					AlwaysIncludedDomains:    []string{"abc.com", "def.com"},
 					NeverIncludedDomains:     []string{"ghi.com", "jkl.com"},
 					FilterExternalToInternal: false,
+					FilterFailedConnections:  false,
 				},
 				HTTPExtensionsFilePath:          "/path/to/http/extensions",
 				BatchSize:                       75000,
@@ -280,6 +282,8 @@ func TestReadFileConfig(t *testing.T) {
 
 			require.Equal(test.expectedConfig.Filter.FilterExternalToInternal, cfg.Filter.FilterExternalToInternal, "FilterExternalToInternal should match expected value")
 
+			require.Equal(test.expectedConfig.Filter.FilterFailedConnections, cfg.Filter.FilterFailedConnections, "FilterFailedConnections should match expected value")
+
 			require.Equal(test.expectedConfig.HTTPExtensionsFilePath, cfg.HTTPExtensionsFilePath, "HTTPExtensionsFilePath should match expected value")
 
 			require.Equal(test.expectedConfig.BatchSize, cfg.BatchSize, "BatchSize should match expected value")

config/filter.go

@@ -22,6 +22,7 @@ type Filter struct {
 	NeverIncludedDomains  []string `json:"never_included_domains"`
 
 	FilterExternalToInternal bool `json:"filter_external_to_internal"`
+	FilterFailedConnections  bool `json:"filter_failed_connections"`
 }
 
 func GetMandatoryNeverIncludeSubnets() []string {
@@ -275,3 +276,7 @@ func (fs *Filter) FilterDomain(domain string) bool {
 func (fs *Filter) CheckIfInternal(host net.IP) bool {
 	return util.ContainsIP(fs.InternalSubnets, host)
 }
+
+func (fs *Filter) FilterFailedConnection(connState string) bool {
+	return fs.FilterFailedConnections && (connState == "S0")
+}

config/filter_test.go

@@ -32,6 +32,9 @@ func TestFilterConnPair(t *testing.T) {
 	// set config filter for external to internal to false
 	cfg.Filter.FilterExternalToInternal = false
 
+	// set config filter for failed connections to false
+	cfg.Filter.FilterFailedConnections = false
+
 	// AlwaysInclude list tests
 	t.Run("AlwaysInclude list tests", func(t *testing.T) {
 		cfg.Filter.AlwaysIncludedSubnets = alwaysIncludedSubnetList
@@ -79,6 +82,20 @@ func TestFilterConnPair(t *testing.T) {
 		checkCases = cfg.Filter.FilterConnPair(net.IP{180, 0, 0, 0}, net.IP{80, 0, 0, 0})
 		require.False(t, checkCases, "filter state should match expected value")
 	})
+
+	t.Run("FailedConnections tests", func(t *testing.T) {
+		cfg.Filter.FilterFailedConnections = true
+
+		checkCases := cfg.Filter.FilterFailedConnection("S0")
+		require.True(t, checkCases, "filter state should match expected value")
+
+		checkCases = cfg.Filter.FilterFailedConnection("S1")
+		require.False(t, checkCases, "filter state should match expected value")
+
+		cfg.Filter.FilterFailedConnections = false
+		checkCases = cfg.Filter.FilterFailedConnection("S0")
+		require.False(t, checkCases, "filter state should match expected value")
+	})
 }
 
 func TestFilterDNSPair(t *testing.T) {

default_config.hjson

@@ -29,7 +29,8 @@
         // connections involving ranges entered into never_included_subnets are filtered out at import time
         never_included_subnets: [], // array of CIDRs
         never_included_domains: [], // array of FQDNs
-        filter_external_to_internal: true // ignores any entries where communication is occurring from an external host to an internal host
+        filter_external_to_internal: true, // ignores any entries where communication is occurring from an external host to an internal host
+        filter_failed_connections: false // ignores any entries where connection attempt seen but no reply (conn & open_conn)
     },
     scoring: {
         beacon: {

importer/conn.go

@@ -122,6 +122,7 @@ func formatConnRecord(cfg *config.Config, parseConn *zeektypes.Conn, importID ut
 	// get source destination pair for connection record
 	src := parseConn.Source
 	dst := parseConn.Destination
+	connState := parseConn.ConnState
 
 	// parse addresses into binary format
 	srcIP := net.ParseIP(src)
@@ -153,7 +154,7 @@ func formatConnRecord(cfg *config.Config, parseConn *zeektypes.Conn, importID ut
 		return nil, err
 	}
 
-	filtered := cfg.Filter.FilterConnPair(srcIP, dstIP)
+	filtered := cfg.Filter.FilterConnPair(srcIP, dstIP) || cfg.Filter.FilterFailedConnection(connState)
 
 	entry := &ConnEntry{
 		ImportTime:  importTime,
@@ -183,7 +184,7 @@ func formatConnRecord(cfg *config.Config, parseConn *zeektypes.Conn, importID ut
 		DstIPBytes:  parseConn.RespIPBytes,
 		SrcPackets:  parseConn.OrigPackets,
 		DstPackets:  parseConn.RespPackets,
-		ConnState:   parseConn.ConnState,
+		ConnState:   connState,
 	}
 
 	// conn is treated differently than the rest of the logs since some other logs might need to correlate

integration/filter_test.go

@@ -701,3 +701,43 @@ func (it *FilterTestSuite) TestFilterExternalToInternal() {
 
 	require.ElementsMatch(t, expectedResData, foundIPs)
 }
+
+// TestFilterFailedConnections
+func (it *FilterTestSuite) TestFilterFailedConnections() {
+	t := it.T()
+	// set up file system interface
+	afs := afero.NewOsFs()
+
+	cfg, err := config.ReadFileConfig(afs, ConfigPath)
+	require.NoError(t, err)
+	cfg.DBConnection = dockerInfo.clickhouseConnection
+	cfg.Filter.FilterFailedConnections = true
+	it.cfg = cfg
+	require.NoError(t, err, "updating config should not return an error")
+
+	require.True(t, cfg.Filter.FilterFailedConnections)
+
+	// import data
+	_, err = cmd.RunImportCmd(time.Now(), cfg, afs, "../test_data/valid_tsv", "filter_fail_conn", false, true)
+	require.NoError(t, err)
+
+	// connect to database
+	db, err := database.ConnectToDB(context.Background(), "filter_fail_conn", cfg, nil)
+	require.NoError(t, err)
+
+	var count uint64
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count(DISTINCT hash) FROM conn
+		WHERE conn_state = 'S0'
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, count, "conn table should contain 0 entries with conn_state = S0, got: %d", count)
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count(DISTINCT hash) FROM openconn
+		WHERE conn_state = 'S0'
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, count, "open conn table should contain 0 entries with conn_state = S0, got: %d", count)
+}