Add filtering option to keep only certain domain records. (dmachard#135)

* Add 'keep domains' filtering option.

Co-authored-by: Leo Vrana <[email protected]>

config.yml

@@ -417,6 +417,10 @@ multiplexer:
 #   drop-fqdn-file: ""
 #   # path file of the domain drop list, domains list can be a partial domain name with regexp expression
 #   drop-domain-file: ""
+#   # path file of the fqdn keep list (all others are dropped), domains list must be a full qualified domain name
+#   keep-fqdn-file: ""
+#   # path file of the domain keep list (all others are dropped), domains list can be a partial domain name with regexp expression
+#   keep-domain-file: ""
 #   # path file of the query IP drop list, one IP address or subnet per line
 #   drop-queryip-file: ""
 #   # path file of the query IP keep list, one IP address or subnet per line

dnsutils/config.go

@@ -96,6 +96,8 @@ type Config struct {
 		Filtering struct {
 			DropFqdnFile    string   `yaml:"drop-fqdn-file"`
 			DropDomainFile  string   `yaml:"drop-domain-file"`
+			KeepFqdnFile    string   `yaml:"keep-fqdn-file"`
+			KeepDomainFile  string   `yaml:"keep-domain-file"`
 			DropQueryIpFile string   `yaml:"drop-queryip-file"`
 			KeepQueryIpFile string   `yaml:"keep-queryip-file"`
 			DropRcodes      []string `yaml:"drop-rcodes,flow"`
@@ -317,6 +319,8 @@ func (c *Config) SetDefault() {
 
 	c.Transformers.Filtering.DropFqdnFile = ""
 	c.Transformers.Filtering.DropDomainFile = ""
+	c.Transformers.Filtering.KeepFqdnFile = ""
+	c.Transformers.Filtering.KeepDomainFile = ""
 	c.Transformers.Filtering.DropQueryIpFile = ""
 	c.Transformers.Filtering.DropRcodes = []string{}
 	c.Transformers.Filtering.LogQueries = true

doc/configuration.md

@@ -231,6 +231,8 @@ This feature can be useful to increase logging performance..
 Options:
 - `drop-fqdn-file`: (string) path file to a fqdn drop list, domains list must be a full qualified domain name
 - `drop-domain-file`: (string) path file to domain drop list, domains list can be a partial domain name with regexp expression
+- `keep-fqdn-file`: (string) path file to a fqdn keep list (all others are dropped), domains list must be a full qualified domain name
+- `keep-domain-file`: (string) path file to domain keep list (all others are dropped), domains list can be a partial domain name with regexp expression
 - `drop-queryip-file`: (string) path file to the query ip or ip prefix drop list
 - `keep-queryip-file`: (string) path file to the query ip or ip prefix keep list, addresses in both drop and keep are always kept
 - `drop-rcodes`: (list of string) rcode list, empty by default
@@ -243,6 +245,8 @@ transforms:
   filtering:
     drop-fqdn-file: ""
     drop-domain-file: ""
+    keep-fqdn-file: ""
+    keep-domain-file: ""
     drop-queryip-file: ""
     keep-queryip-file: ""
     drop-rcodes: []

testsdata/filtering_keep_domains.txt

@@ -0,0 +1,2 @@
+google.fr
+test.github.com

testsdata/filtering_keep_domains_regex.txt

@@ -0,0 +1,3 @@
+(mail|sheets)\.google\.com$
+test\.github\.com$
+.+\.google\.com$

transformers/filtering.go

@@ -14,18 +14,21 @@ import (
 )
 
 type FilteringProcessor struct {
-	config           *dnsutils.Config
-	logger           *logger.Logger
-	dropDomains      bool
-	mapRcodes        map[string]bool
-	ipsetDrop        *netaddr.IPSet
-	ipsetKeep        *netaddr.IPSet
-	listFqdns        map[string]bool
-	listDomainsRegex map[string]*regexp.Regexp
-	fileWatcher      *fsnotify.Watcher
-	name             string
-	downsample       int
-	downsampleCount  int
+	config                *dnsutils.Config
+	logger                *logger.Logger
+	dropDomains           bool
+	keepDomains           bool
+	mapRcodes             map[string]bool
+	ipsetDrop             *netaddr.IPSet
+	ipsetKeep             *netaddr.IPSet
+	listFqdns             map[string]bool
+	listDomainsRegex      map[string]*regexp.Regexp
+	listKeepFqdns         map[string]bool
+	listKeepDomainsRegex  map[string]*regexp.Regexp
+	fileWatcher           *fsnotify.Watcher
+	name                  string
+	downsample            int
+	downsampleCount       int
 }
 
 func NewFilteringProcessor(config *dnsutils.Config, logger *logger.Logger, name string) FilteringProcessor {
@@ -37,15 +40,17 @@ func NewFilteringProcessor(config *dnsutils.Config, logger *logger.Logger, name
 	defer watcher.Close()
 
 	d := FilteringProcessor{
-		config:           config,
-		logger:           logger,
-		mapRcodes:        make(map[string]bool),
-		ipsetDrop:        &netaddr.IPSet{},
-		ipsetKeep:        &netaddr.IPSet{},
-		listFqdns:        make(map[string]bool),
-		listDomainsRegex: make(map[string]*regexp.Regexp),
-		fileWatcher:      watcher,
-		name:             name,
+		config:                config,
+		logger:                logger,
+		mapRcodes:             make(map[string]bool),
+		ipsetDrop:             &netaddr.IPSet{},
+		ipsetKeep:             &netaddr.IPSet{},
+		listFqdns:             make(map[string]bool),
+		listDomainsRegex:      make(map[string]*regexp.Regexp),
+		listKeepFqdns:         make(map[string]bool),
+		listKeepDomainsRegex:  make(map[string]*regexp.Regexp),
+		fileWatcher:           watcher,
+		name:                  name,
 	}
 
 	d.LoadRcodes()
@@ -170,8 +175,40 @@ func (p *FilteringProcessor) LoadDomainsList() {
 			p.LogInfo("loaded with %d domains to the drop list", len(p.listDomainsRegex))
 			p.dropDomains = true
 		}
-
 	}
+
+	if len(p.config.Transformers.Filtering.KeepFqdnFile) > 0 {
+		file, err := os.Open(p.config.Transformers.Filtering.KeepFqdnFile)
+		if err != nil {
+			p.LogError("unable to open KeepFqdnFile file: ", err)
+			p.keepDomains = false 
+		} else {
+			scanner := bufio.NewScanner(file)
+			for scanner.Scan() {
+				keepDomain := strings.ToLower(scanner.Text())
+				p.listKeepFqdns[keepDomain] = true 
+			}
+			p.LogInfo("loaded with %d fqdns to the keep list", len(p.listKeepFqdns))
+			p.keepDomains = true
+		}
+	} 
+
+	if len(p.config.Transformers.Filtering.KeepDomainFile) > 0 {
+		file, err := os.Open(p.config.Transformers.Filtering.KeepDomainFile)
+		if err != nil {
+			p.LogError("unable to open KeepDomainFile file: ", err)
+			p.keepDomains = false 
+		} else {
+			scanner := bufio.NewScanner(file)
+			for scanner.Scan() {
+				keepDomain := strings.ToLower(scanner.Text())
+				p.listKeepDomainsRegex[keepDomain] = regexp.MustCompile(keepDomain) 
+			}
+			p.LogInfo("loaded with %d domains to the keep list", len(p.listKeepDomainsRegex))
+			p.keepDomains = true
+		}
+	} 
+
 }
 
 func (p *FilteringProcessor) LogInfo(msg string, v ...interface{}) {
@@ -236,6 +273,23 @@ func (p *FilteringProcessor) CheckIfDrop(dm *dnsutils.DnsMessage) bool {
 		}
 	}
 
+	// only certain domains ? read list but flip the responses.
+	if p.keepDomains {
+		// only 
+		if _, ok := p.listKeepFqdns[dm.DNS.Qname]; ok {
+			return false 
+		}
+
+		// partiel fqdn with regexp
+		for _, r := range p.listKeepDomainsRegex {
+			if r.MatchString(dm.DNS.Qname) {
+				return false
+			}
+		}
+
+		return true
+	} 
+
 	if p.downsample > 0 {
 		p.downsampleCount += 1
 		if p.downsampleCount % p.downsample != 0 {

transformers/filtering_test.go

@@ -139,16 +139,85 @@ func TestFilteringByDomainRegex(t *testing.T) {
 	}
 }
 
-func TestFilteringByDownsample(t *testing.T) {
+func TestFilteringByKeepDomain(t *testing.T) {
 	// config
 	config := dnsutils.GetFakeConfig()
-	config.Transformers.Filtering.Downsample = 2
+
+	// file contains google.fr, test.github.com
+	config.Transformers.Filtering.KeepDomainFile = "../testsdata/filtering_keep_domains.txt"
+
+	// init subproccesor
+	filtering := NewFilteringProcessor(config, logger.New(false), "test")
+
+	dm := dnsutils.GetFakeDnsMessage()
+	dm.DNS.Qname = "mail.google.com"
+	if filtering.CheckIfDrop(&dm) == false {
+		t.Errorf("dns query should be dropped! Domain: %s", dm.DNS.Qname)
+	}
+
+	dm.DNS.Qname = "example.com"
+	if filtering.CheckIfDrop(&dm) == false {
+		t.Errorf("dns query should not be dropped!")
+	}
+
+	dm.DNS.Qname = "test.github.com"
+	if filtering.CheckIfDrop(&dm) == true {
+		t.Errorf("dns query should not be dropped!")
+	}
+
+	dm.DNS.Qname = "google.fr"
+	if filtering.CheckIfDrop(&dm) == true {
+		t.Errorf("dns query should not be dropped!")
+	}
+}
+
+func TestFilteringByKeepDomainRegex(t *testing.T) {
+	// config
+	config := dnsutils.GetFakeConfig()
+
+	/* file contains:
+	(mail|sheets).google.com$
+	test.github.com$
+	.+.google.com$
+	*/
+	config.Transformers.Filtering.KeepDomainFile = "../testsdata/filtering_keep_domains_regex.txt"
 
 	// init subproccesor
 	filtering := NewFilteringProcessor(config, logger.New(false), "test")
 
 	dm := dnsutils.GetFakeDnsMessage()
+	dm.DNS.Qname = "mail.google.com"
+	if filtering.CheckIfDrop(&dm) == true {
+		t.Errorf("dns query should not be dropped!")
+	}
+
+	dm.DNS.Qname = "test.google.com.ru"
+	if filtering.CheckIfDrop(&dm) == false {
+
+        // If this passes then these are not terminated.
+		t.Errorf("dns query should be dropped!")
+	}
+
+	dm.DNS.Qname = "test.github.com"
+	if filtering.CheckIfDrop(&dm) == true {
+		t.Errorf("dns query should not be dropped!")
+	}
 
+	dm.DNS.Qname = "test.github.com.malware.ru"
+	if filtering.CheckIfDrop(&dm) == false {
+		t.Errorf("dns query should be dropped!")
+	}
+}
+
+func TestFilteringByDownsample(t *testing.T) {
+	// config
+	config := dnsutils.GetFakeConfig()
+	config.Transformers.Filtering.Downsample = 2
+
+	// init subproccesor
+	filtering := NewFilteringProcessor(config, logger.New(false), "test")
+	dm := dnsutils.GetFakeDnsMessage()
+
 	// filtering.downsampleCount 
 	if filtering.CheckIfDrop(&dm) == false {
 		t.Errorf("dns query should be dropped! downsampled should exclude first hit.")
@@ -177,4 +246,4 @@ func TestFilteringByDownsample(t *testing.T) {
 		t.Errorf("dns query should not be dropped! downsampling rate is set to 0 and should not downsample.")
 	}
 
-}
+}