diff --git a/config.hjson b/config.hjson index fe3cd3c..18353ae 100644 --- a/config.hjson +++ b/config.hjson @@ -2,6 +2,7 @@ update_check_enabled: true, filtering: { filter_external_to_internal: true, + filter_failed_connections: false, internal_subnets: ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fd00::/8"] }, threat_intel: { diff --git a/config/config.go b/config/config.go index b34f64b..c8acdc2 100644 --- a/config/config.go +++ b/config/config.go @@ -543,6 +543,7 @@ func defaultConfig() Config { AlwaysIncludedDomains: []string{}, NeverIncludedDomains: []string{}, FilterExternalToInternal: true, + FilterFailedConnections: false, }, HTTPExtensionsFilePath: "./http_extensions_list.csv", BatchSize: 100000, diff --git a/config/config_test.go b/config/config_test.go index 34070f6..fb1f61c 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -54,6 +54,7 @@ func TestReadFileConfig(t *testing.T) { always_included_domains: ["abc.com", "def.com"], never_included_domains: ["ghi.com", "jkl.com"], filter_external_to_internal: false, + filter_failed_connections: false, }, http_extensions_file_path: "/path/to/http/extensions", batch_size: 75000, @@ -151,6 +152,7 @@ func TestReadFileConfig(t *testing.T) { AlwaysIncludedDomains: []string{"abc.com", "def.com"}, NeverIncludedDomains: []string{"ghi.com", "jkl.com"}, FilterExternalToInternal: false, + FilterFailedConnections: false, }, HTTPExtensionsFilePath: "/path/to/http/extensions", BatchSize: 75000, @@ -280,6 +282,8 @@ func TestReadFileConfig(t *testing.T) { require.Equal(test.expectedConfig.Filter.FilterExternalToInternal, cfg.Filter.FilterExternalToInternal, "FilterExternalToInternal should match expected value") + require.Equal(test.expectedConfig.Filter.FilterFailedConnections, cfg.Filter.FilterFailedConnections, "FilterFailedConnections should match expected value") + require.Equal(test.expectedConfig.HTTPExtensionsFilePath, cfg.HTTPExtensionsFilePath, "HTTPExtensionsFilePath should match expected value") require.Equal(test.expectedConfig.BatchSize, cfg.BatchSize, "BatchSize should match expected value") diff --git a/config/filter.go b/config/filter.go index c38b674..bbac705 100644 --- a/config/filter.go +++ b/config/filter.go @@ -22,6 +22,7 @@ type Filter struct { NeverIncludedDomains []string `json:"never_included_domains"` FilterExternalToInternal bool `json:"filter_external_to_internal"` + FilterFailedConnections bool `json:"filter_failed_connections"` } func GetMandatoryNeverIncludeSubnets() []string { @@ -275,3 +276,7 @@ func (fs *Filter) FilterDomain(domain string) bool { func (fs *Filter) CheckIfInternal(host net.IP) bool { return util.ContainsIP(fs.InternalSubnets, host) } + +func (fs *Filter) FilterFailedConnection(connState string) bool { + return fs.FilterFailedConnections && (connState == "S0") +} diff --git a/config/filter_test.go b/config/filter_test.go index 5dbe09a..140f0d7 100644 --- a/config/filter_test.go +++ b/config/filter_test.go @@ -32,6 +32,9 @@ func TestFilterConnPair(t *testing.T) { // set config filter for external to internal to false cfg.Filter.FilterExternalToInternal = false + // set config filter for failed connections to false + cfg.Filter.FilterFailedConnections = false + // AlwaysInclude list tests t.Run("AlwaysInclude list tests", func(t *testing.T) { cfg.Filter.AlwaysIncludedSubnets = alwaysIncludedSubnetList @@ -79,6 +82,20 @@ func TestFilterConnPair(t *testing.T) { checkCases = cfg.Filter.FilterConnPair(net.IP{180, 0, 0, 0}, net.IP{80, 0, 0, 0}) require.False(t, checkCases, "filter state should match expected value") }) + + t.Run("FailedConnections tests", func(t *testing.T) { + cfg.Filter.FilterFailedConnections = true + + checkCases := cfg.Filter.FilterFailedConnection("S0") + require.True(t, checkCases, "filter state should match expected value") + + checkCases = cfg.Filter.FilterFailedConnection("S1") + require.False(t, checkCases, "filter state should match expected value") + + cfg.Filter.FilterFailedConnections = false + checkCases = cfg.Filter.FilterFailedConnection("S0") + require.False(t, checkCases, "filter state should match expected value") + }) } func TestFilterDNSPair(t *testing.T) { diff --git a/default_config.hjson b/default_config.hjson index 33a9013..68ac162 100644 --- a/default_config.hjson +++ b/default_config.hjson @@ -29,7 +29,8 @@ // connections involving ranges entered into never_included_subnets are filtered out at import time never_included_subnets: [], // array of CIDRs never_included_domains: [], // array of FQDNs - filter_external_to_internal: true // ignores any entries where communication is occurring from an external host to an internal host + filter_external_to_internal: true, // ignores any entries where communication is occurring from an external host to an internal host + filter_failed_connections: false // ignores any entries where connection attempt seen but no reply (conn & open_conn) }, scoring: { beacon: { diff --git a/importer/conn.go b/importer/conn.go index b5ec1c9..0a1796a 100644 --- a/importer/conn.go +++ b/importer/conn.go @@ -122,6 +122,7 @@ func formatConnRecord(cfg *config.Config, parseConn *zeektypes.Conn, importID ut // get source destination pair for connection record src := parseConn.Source dst := parseConn.Destination + connState := parseConn.ConnState // parse addresses into binary format srcIP := net.ParseIP(src) @@ -153,7 +154,7 @@ func formatConnRecord(cfg *config.Config, parseConn *zeektypes.Conn, importID ut return nil, err } - filtered := cfg.Filter.FilterConnPair(srcIP, dstIP) + filtered := cfg.Filter.FilterConnPair(srcIP, dstIP) || cfg.Filter.FilterFailedConnection(connState) entry := &ConnEntry{ ImportTime: importTime, @@ -183,7 +184,7 @@ func formatConnRecord(cfg *config.Config, parseConn *zeektypes.Conn, importID ut DstIPBytes: parseConn.RespIPBytes, SrcPackets: parseConn.OrigPackets, DstPackets: parseConn.RespPackets, - ConnState: parseConn.ConnState, + ConnState: connState, } // conn is treated differently than the rest of the logs since some other logs might need to correlate diff --git a/integration/filter_test.go b/integration/filter_test.go index 5a7ebcd..5037428 100644 --- a/integration/filter_test.go +++ b/integration/filter_test.go @@ -701,3 +701,43 @@ func (it *FilterTestSuite) TestFilterExternalToInternal() { require.ElementsMatch(t, expectedResData, foundIPs) } + +// TestFilterFailedConnections +func (it *FilterTestSuite) TestFilterFailedConnections() { + t := it.T() + // set up file system interface + afs := afero.NewOsFs() + + cfg, err := config.ReadFileConfig(afs, ConfigPath) + require.NoError(t, err) + cfg.DBConnection = dockerInfo.clickhouseConnection + cfg.Filter.FilterFailedConnections = true + it.cfg = cfg + require.NoError(t, err, "updating config should not return an error") + + require.True(t, cfg.Filter.FilterFailedConnections) + + // import data + _, err = cmd.RunImportCmd(time.Now(), cfg, afs, "../test_data/valid_tsv", "filter_fail_conn", false, true) + require.NoError(t, err) + + // connect to database + db, err := database.ConnectToDB(context.Background(), "filter_fail_conn", cfg, nil) + require.NoError(t, err) + + var count uint64 + + err = db.Conn.QueryRow(db.GetContext(), ` + SELECT count(DISTINCT hash) FROM conn + WHERE conn_state = 'S0' + `).Scan(&count) + require.NoError(t, err) + require.EqualValues(t, 0, count, "conn table should contain 0 entries with conn_state = S0, got: %d", count) + + err = db.Conn.QueryRow(db.GetContext(), ` + SELECT count(DISTINCT hash) FROM openconn + WHERE conn_state = 'S0' + `).Scan(&count) + require.NoError(t, err) + require.EqualValues(t, 0, count, "open conn table should contain 0 entries with conn_state = S0, got: %d", count) +}