Remove beaconfqdn package (#771)

* Remove beaconfqdn package * update reporting/templates/templates.go Co-authored-by: Logan L <[email protected]>
activecm · Dec 19, 2022 · 60a433a · 60a433a
1 parent 1870a54
commit 60a433a
Show file tree

Hide file tree

Showing 23 changed files with 19 additions and 2,236 deletions.
diff --git a/commands/clean.go b/commands/clean.go
@@ -37,7 +37,6 @@ func cleanDatabase(c *cli.Context) error {
 		res.Config.T.Structure.UniqueConnProxyTable: "Uconn Proxy Analysis",
 		res.Config.T.BeaconProxy.BeaconProxyTable:   "Proxy Beacon Analysis",
 		res.Config.T.Beacon.BeaconTable:             "Beacon Analysis",
-		res.Config.T.BeaconFQDN.BeaconFQDNTable:     "FQDN Beacon Analysis",
 		res.Config.T.Structure.SNIConnTable:         "SNI Beacon Analysis",
 		res.Config.T.BeaconSNI.BeaconSNITable:       "SNI Connection Analysis",
 		res.Config.T.UserAgent.UserAgentTable:       "UserAgent Analysis",

diff --git a/commands/show-beacons-fqdn.go b/commands/show-beacons-fqdn.go
diff --git a/config/static.go b/config/static.go
@@ -31,7 +31,6 @@ type (
 		Log          LogStaticCfg         `yaml:"LogConfig"`
 		Blacklisted  BlacklistedStaticCfg `yaml:"BlackListed"`
 		Beacon       BeaconStaticCfg      `yaml:"Beacon"`
-		BeaconFQDN   BeaconFQDNStaticCfg  `yaml:"BeaconFQDN"`
 		BeaconProxy  BeaconProxyStaticCfg `yaml:"BeaconProxy"`
 		BeaconSNI    BeaconSNIStaticCfg   `yaml:"BeaconSNI"`
 		DNS          DNSStaticCfg         `yaml:"DNS"`
@@ -104,12 +103,6 @@ type (
 		HistWeight              float64 `yaml:"HistogramScoreWeight" default:"0.25"`
 	}
 
-	//BeaconFQDNStaticCfg is used to control the fqdn beaconing analysis module
-	BeaconFQDNStaticCfg struct {
-		Enabled                 bool `yaml:"Enabled" default:"true"`
-		DefaultConnectionThresh int  `yaml:"DefaultConnectionThresh" default:"20"`
-	}
-
 	//BeaconProxyStaticCfg is used to control the proxy beaconing analysis module
 	BeaconProxyStaticCfg struct {
 		Enabled                 bool `yaml:"Enabled" default:"true"`

diff --git a/config/static_test.go b/config/static_test.go
@@ -34,9 +34,6 @@ DNS:
 Beacon:
     Enabled: true
     DefaultConnectionThresh: 24
-BeaconFQDN:
-    Enabled: true
-    DefaultConnectionThresh: 24
 BeaconProxy:
     Enabled: true
     DefaultConnectionThresh: 24
@@ -85,10 +82,6 @@ var testConfigFullExp = StaticCfg{
 		Enabled:                 true,
 		DefaultConnectionThresh: 24,
 	},
-	BeaconFQDN: BeaconFQDNStaticCfg{
-		Enabled:                 true,
-		DefaultConnectionThresh: 24,
-	},
 	BeaconProxy: BeaconProxyStaticCfg{
 		Enabled:                 true,
 		DefaultConnectionThresh: 24,

diff --git a/config/tables.go b/config/tables.go
@@ -8,7 +8,6 @@ type (
 		Structure   StructureTableCfg
 		Beacon      BeaconTableCfg
 		BeaconSNI   BeaconSNITableCfg
-		BeaconFQDN  BeaconFQDNTableCfg
 		BeaconProxy BeaconProxyTableCfg
 		UserAgent   UserAgentTableCfg
 		Cert        CertificateTableCfg
@@ -49,11 +48,6 @@ type (
 		BeaconSNITable string `default:"beaconSNI"`
 	}
 
-	//BeaconFQDNTableCfg is used to control the beaconing analysis module
-	BeaconFQDNTableCfg struct {
-		BeaconFQDNTable string `default:"beaconFQDN"`
-	}
-
 	//BeaconProxyTableCfg is used to control the beaconing analysis module
 	BeaconProxyTableCfg struct {
 		BeaconProxyTable string `default:"beaconProxy"`

diff --git a/config/testing.go b/config/testing.go
@@ -32,9 +32,6 @@ DNS:
 Beacon:
     Enabled: true
     DefaultConnectionThresh: 24
-BeaconFQDN:
-    Enabled: true
-    DefaultConnectionThresh: 24
 BeaconProxy:
     Enabled: true
     DefaultConnectionThresh: 24

diff --git a/etc/rita.yaml b/etc/rita.yaml
@@ -150,19 +150,6 @@ Beacon:
   DurationScoreWeight: 0.25
   HistogramScoreWeight: 0.25
 
-BeaconFQDN:
-  Enabled: true
-  # The default minimum number of connections used for beacons FQDN analysis.
-  # Any two hosts connecting fewer than this number will not be analyzed.
-  # 20 was chosen as it is a little bit less than once per hour in a day,
-  # and allows for any packet loss that could occur.
-
-  # If you choose a lower value, this will significantly increase both
-  # the analysis time and the number of false positives. You can safely
-  # increase this value to improve performance if you are not concerned
-  # about slow beacons.
-  DefaultConnectionThresh: 20
-
 BeaconSNI:
   Enabled: true
   # The default minimum number of connections used for beacons SNI analysis.

diff --git a/parser/fsimporter.go b/parser/fsimporter.go
@@ -13,7 +13,6 @@ import (
 	"github.com/activecm/rita/parser/files"
 	"github.com/activecm/rita/parser/parsetypes"
 	"github.com/activecm/rita/pkg/beacon"
-	"github.com/activecm/rita/pkg/beaconfqdn"
 	"github.com/activecm/rita/pkg/beaconproxy"
 	"github.com/activecm/rita/pkg/beaconsni"
 	"github.com/activecm/rita/pkg/blacklist"
@@ -55,7 +54,7 @@ type (
 	}
 )
 
-//NewFSImporter creates a new file system importer
+// NewFSImporter creates a new file system importer
 func NewFSImporter(res *resources.Resources) *FSImporter {
 	// set batchSize to the max of 4GB or a half of system RAM to prevent running out of memory while importing
 	batchSize := int64(util.MaxUint64(4*(1<<30), (memory.TotalMemory() / 2)))
@@ -74,12 +73,12 @@ var trustedAppReferenceList = [...]trustedAppTiplet{
 	{"tcp", 443, "ssl"},
 }
 
-//GetInternalSubnets returns the internal subnets from the config file
+// GetInternalSubnets returns the internal subnets from the config file
 func (fs *FSImporter) GetInternalSubnets() []*net.IPNet {
 	return fs.internal
 }
 
-//CollectFileDetails reads and hashes the files
+// CollectFileDetails reads and hashes the files
 func (fs *FSImporter) CollectFileDetails(importFiles []string, threads int) []*files.IndexedFile {
 	// find all of the potential bro log paths
 	logFiles := files.GatherLogFiles(importFiles, fs.log)
@@ -90,7 +89,7 @@ func (fs *FSImporter) CollectFileDetails(importFiles []string, threads int) []*f
 	)
 }
 
-//Run starts the importing
+// Run starts the importing
 func (fs *FSImporter) Run(indexedFiles []*files.IndexedFile, threads int) {
 	start := time.Now()
 
@@ -197,9 +196,6 @@ func (fs *FSImporter) Run(indexedFiles []*files.IndexedFile, threads int) {
 		// build or update Beacons table
 		fs.buildBeacons(retVals.UniqueConnMap, retVals.HostMap, minTimestamp, maxTimestamp)
 
-		// build or update the FQDN Beacons Table
-		fs.buildFQDNBeacons(retVals.HostMap, minTimestamp, maxTimestamp)
-
 		// build or update the Proxy Beacons Table
 		fs.buildProxyBeacons(retVals.ProxyUniqueConnMap, retVals.HostMap, minTimestamp, maxTimestamp)
 
@@ -307,10 +303,10 @@ func batchFilesBySize(indexedFiles []*files.IndexedFile, size int64) [][]*files.
 	return batches
 }
 
-//parseFiles takes in a list of indexed bro files, the number of
-//threads to use to parse the files, whether or not to sort data by date,
-//a MongoDB datastore object to store the bro data in, and a logger to report
-//errors and parses the bro files line by line into the database.
+// parseFiles takes in a list of indexed bro files, the number of
+// threads to use to parse the files, whether or not to sort data by date,
+// a MongoDB datastore object to store the bro data in, and a logger to report
+// errors and parses the bro files line by line into the database.
 func (fs *FSImporter) parseFiles(indexedFiles []*files.IndexedFile, parsingThreads int, logger *log.Logger) ParseResults {
 
 	fmt.Println("\t[-] Parsing logs to: " + fs.database.GetSelectedDB() + " ... ")
@@ -413,7 +409,7 @@ func (fs *FSImporter) parseFiles(indexedFiles []*files.IndexedFile, parsingThrea
 	return retVals
 }
 
-//buildExplodedDNS .....
+// buildExplodedDNS .....
 func (fs *FSImporter) buildExplodedDNS(domainMap map[string]int) {
 
 	if fs.config.S.DNS.Enabled {
@@ -431,7 +427,7 @@ func (fs *FSImporter) buildExplodedDNS(domainMap map[string]int) {
 	}
 }
 
-//buildCertificates .....
+// buildCertificates .....
 func (fs *FSImporter) buildCertificates(certMap map[string]*certificate.Input) {
 
 	if len(certMap) > 0 {
@@ -448,7 +444,7 @@ func (fs *FSImporter) buildCertificates(certMap map[string]*certificate.Input) {
 
 }
 
-//removeAnalysisChunk .....
+// removeAnalysisChunk .....
 func (fs *FSImporter) removeAnalysisChunk(cid int) error {
 
 	// Set up the remover
@@ -465,7 +461,7 @@ func (fs *FSImporter) removeAnalysisChunk(cid int) error {
 
 }
 
-//buildHostnames .....
+// buildHostnames .....
 func (fs *FSImporter) buildHostnames(hostnameMap map[string]*hostname.Input) {
 	// non-optional module
 	if len(hostnameMap) > 0 {
@@ -589,27 +585,6 @@ func (fs *FSImporter) buildBeacons(uconnMap map[string]*uconn.Input, hostMap map
 
 }
 
-func (fs *FSImporter) buildFQDNBeacons(hostMap map[string]*host.Input, minTimestamp, maxTimestamp int64) {
-	if fs.config.S.BeaconFQDN.Enabled {
-		if len(hostMap) > 0 {
-			beaconFQDNRepo := beaconfqdn.NewMongoRepository(fs.database, fs.config, fs.log)
-
-			err := beaconFQDNRepo.CreateIndexes()
-			if err != nil {
-				fs.log.Error(err)
-			}
-
-			// Send the list of hosts out to the FQDN beacon analysis pkg.
-			// The list of external hosts seen in the current set of logs determines
-			// which FQDN beacons need to be updated.
-			beaconFQDNRepo.Upsert(hostMap, minTimestamp, maxTimestamp)
-		} else {
-			fmt.Println("\t[!] No FQDN Beacon data to analyze")
-		}
-	}
-
-}
-
 func (fs *FSImporter) buildProxyBeacons(uconnProxyMap map[string]*uconnproxy.Input, hostMap map[string]*host.Input, minTimestamp, maxTimestamp int64) {
 	if fs.config.S.BeaconProxy.Enabled {
 		if len(uconnProxyMap) > 0 {
@@ -647,7 +622,7 @@ func (fs *FSImporter) buildSNIBeacons(tlsMap map[string]*sniconn.TLSInput, httpM
 	}
 }
 
-//buildUserAgent .....
+// buildUserAgent .....
 func (fs *FSImporter) buildUserAgent(useragentMap map[string]*useragent.Input) {
 
 	if fs.config.S.UserAgent.Enabled {