sink dnstap to a file - feature request dmachard#176 (dmachard#183)

README.md

@@ -16,30 +16,30 @@ DNS-collector also contains DNS parser with [`EDNS`](doc/dnsparser.md) support.
 - *Listen for logging traffic with streaming network protocols*
     - Protobuf [`DNStap`](doc/collectors.md#dns-tap) with tcp or unix support
     - Protobuf [`PowerDNS`](doc/collectors.md#protobuf-powerdns) streams
-    - DNStap [`Proxifier`](doc/collectors.md#dnstap-relay) without decoding
+    - [`Proxifier`](doc/collectors.md#dns-tap-proxifier) for DNSTap streams
 - *Live capture on a network interface*   
     - [`AF_PACKET`](doc/collectors.md#dns-sniffer) socket with BPF filter
 - *Read text or binary files as input*
     - Read and tail on [`Plain text`](doc/collectors.md#tail) files
-    - Ingest [`PCAP`](doc/collectors.md#ingest-pcap) files by watching a directory
+    - Ingest [`PCAP`](doc/collectors.md#file-ingestor) or [`DNSTap`](doc/collectors.md#file-ingestor) files by watching a directory
 
 **Loggers**:
-- *Redirect DNS logs to stdout or files in plain text or binary mode*:
+- *Redirect DNS logs to stdout or files in plain text or binary mode*
     - Print directly to your [`Stdout`](doc/loggers.md#stdout) console
-    - Write to [`File`](doc/loggers.md#log-file) 
-        - with custom [Text](doc/configuration.md#custom-text-format) format
-        - [Json](doc/dnsjson.md) encoding
-        - [Pcap](doc/loggers.md#log-file) format
-
-- *Provide metrics and API*:
+    - Write to [`File`](doc/loggers.md#log-file) with several formats
+        - [Custom Text](doc/configuration.md#custom-text-format)
+        - [Json](doc/dnsjson.md)
+        - [Pcap](doc/loggers.md#log-file)
+        - [Dnstap](doc/loggers.md#log-file)
+- *Provide metrics and API*
     - [`Prometheus`](doc/loggers.md#prometheus) metrics and visualize-it with built-in [dashboards](doc/dashboards.md) for Grafana
     - [`Statsd`](doc/loggers.md#statsd-client) support
     - [`REST API`](doc/loggers.md#rest-api) with [swagger](https://generator.swagger.io/?url=https://raw.githubusercontent.com/dmachard/go-dnscollector/main/doc/swagger.yml) to search DNS domains
-- *Send to remote host with generic protocol*:
+- *Send to remote host with generic protocol*
     - [`TCP`](doc/loggers.md#tcp-client)
     - [`Syslog`](doc/loggers.md#syslog)
     - [`DNSTap`](doc/loggers.md#dnstap-client) protobuf messages
-- *Send to various sinks*:
+- *Send to various sinks*
     - [`Fluentd`](doc/loggers.md#fluentd-client)
     - [`InfluxDB`](doc/loggers.md#influxdb-client)
     - [`Loki`](doc/loggers.md#loki-client)
@@ -49,19 +49,19 @@ DNS-collector also contains DNS parser with [`EDNS`](doc/dnsparser.md) support.
 
 - [`Traffic filtering`](doc/configuration.md#dns-filtering)
     - Downsampling
-    - Dropping Qname, QueryIP or Rcode
+    - Dropping per Qname, QueryIP or Rcode
 - [`User Privacy`](doc/configuration.md#user-privacy)
     - Anonymize QueryIP
     - Minimaze Qname
 - [`Normalize Qname`](doc/configuration.md#normalize)
-    - lowercase
+    - To lowercase
     - Get TLD and TLD+1
 - [`Geographical metadata`](doc/configuration.md#geoip-support)
     - Country and City
 - [`Suspicious traffic detector`](doc/configuration.md#suspicious)
     - Malformed and large packet
     - Uncommon Qtypes used
-    - Unallowed Chars in Qname
+    - Unallowed chars in Qname
     - Excessive number of labels
     - Long Qname
 

collectors/dnssniffer_windows.go

@@ -37,6 +37,7 @@ func (c *DnsSniffer) GetName() string { return c.name }
 func (c *DnsSniffer) SetLoggers(loggers []dnsutils.Worker) {
 	c.loggers = loggers
 }
+
 func (c *DnsSniffer) LogInfo(msg string, v ...interface{}) {
 	c.logger.Info("["+c.name+"] collector dns sniffer - "+msg, v...)
 }

collectors/dnstap.go

@@ -76,8 +76,8 @@ func (c *Dnstap) HandleConn(conn net.Conn) {
 	c.LogInfo("new connection from %s\n", peer)
 
 	// start dnstap subprocessor
-	dnstap_subprocessor := NewDnstapProcessor(c.config, c.logger, c.name)
-	go dnstap_subprocessor.Run(c.Loggers())
+	dnstapProcessor := NewDnstapProcessor(c.config, c.logger, c.name)
+	go dnstapProcessor.Run(c.Loggers())
 
 	// frame stream library
 	r := bufio.NewReader(conn)
@@ -93,12 +93,12 @@ func (c *Dnstap) HandleConn(conn net.Conn) {
 	}
 
 	// process incoming frame and send it to dnstap consumer channel
-	if err := fs.ProcessFrame(dnstap_subprocessor.GetChannel()); err != nil {
+	if err := fs.ProcessFrame(dnstapProcessor.GetChannel()); err != nil {
 		c.LogError("transport error: %s", err)
 	}
 
 	// stop all subprocessors
-	dnstap_subprocessor.Stop()
+	dnstapProcessor.Stop()
 
 	c.LogInfo("%s - connection closed\n", peer)
 }

collectors/dnstaprelay.go → collectors/dnstap_proxifier.go

@@ -13,7 +13,7 @@ import (
 	"github.com/dmachard/go-logger"
 )
 
-type DnstapRelay struct {
+type DnstapProxifier struct {
 	done     chan bool
 	listen   net.Listener
 	conns    []net.Conn
@@ -24,9 +24,9 @@ type DnstapRelay struct {
 	name     string
 }
 
-func NewDnstapRelay(loggers []dnsutils.Worker, config *dnsutils.Config, logger *logger.Logger, name string) *DnstapRelay {
+func NewDnstapProxifier(loggers []dnsutils.Worker, config *dnsutils.Config, logger *logger.Logger, name string) *DnstapProxifier {
 	logger.Info("[%s] dnstap relay collector - enabled", name)
-	s := &DnstapRelay{
+	s := &DnstapProxifier{
 		done:    make(chan bool),
 		config:  config,
 		loggers: loggers,
@@ -37,37 +37,37 @@ func NewDnstapRelay(loggers []dnsutils.Worker, config *dnsutils.Config, logger *
 	return s
 }
 
-func (c *DnstapRelay) GetName() string { return c.name }
+func (c *DnstapProxifier) GetName() string { return c.name }
 
-func (c *DnstapRelay) SetLoggers(loggers []dnsutils.Worker) {
+func (c *DnstapProxifier) SetLoggers(loggers []dnsutils.Worker) {
 	c.loggers = loggers
 }
 
-func (c *DnstapRelay) Loggers() []chan dnsutils.DnsMessage {
+func (c *DnstapProxifier) Loggers() []chan dnsutils.DnsMessage {
 	channels := []chan dnsutils.DnsMessage{}
 	for _, p := range c.loggers {
 		channels = append(channels, p.Channel())
 	}
 	return channels
 }
 
-func (c *DnstapRelay) ReadConfig() {
-	if !dnsutils.IsValidTLS(c.config.Collectors.DnstapRelay.TlsMinVersion) {
+func (c *DnstapProxifier) ReadConfig() {
+	if !dnsutils.IsValidTLS(c.config.Collectors.DnstapProxifier.TlsMinVersion) {
 		c.logger.Fatal("collector dnstap relay - invalid tls min version")
 	}
 
-	c.sockPath = c.config.Collectors.DnstapRelay.SockPath
+	c.sockPath = c.config.Collectors.DnstapProxifier.SockPath
 }
 
-func (c *DnstapRelay) LogInfo(msg string, v ...interface{}) {
+func (c *DnstapProxifier) LogInfo(msg string, v ...interface{}) {
 	c.logger.Info("["+c.name+"] dnstap collector relay - "+msg, v...)
 }
 
-func (c *DnstapRelay) LogError(msg string, v ...interface{}) {
+func (c *DnstapProxifier) LogError(msg string, v ...interface{}) {
 	c.logger.Error("["+c.name+"] dnstap collector relay - "+msg, v...)
 }
 
-func (c *DnstapRelay) HandleFrame(recvFrom chan []byte, sendTo []chan dnsutils.DnsMessage) {
+func (c *DnstapProxifier) HandleFrame(recvFrom chan []byte, sendTo []chan dnsutils.DnsMessage) {
 	for data := range recvFrom {
 		// init DNS message container
 		dm := dnsutils.DnsMessage{}
@@ -83,7 +83,7 @@ func (c *DnstapRelay) HandleFrame(recvFrom chan []byte, sendTo []chan dnsutils.D
 	}
 }
 
-func (c *DnstapRelay) HandleConn(conn net.Conn) {
+func (c *DnstapProxifier) HandleConn(conn net.Conn) {
 	// close connection on function exit
 	defer conn.Close()
 
@@ -117,11 +117,11 @@ func (c *DnstapRelay) HandleConn(conn net.Conn) {
 	c.LogInfo("%s - connection closed\n", peer)
 }
 
-func (c *DnstapRelay) Channel() chan dnsutils.DnsMessage {
+func (c *DnstapProxifier) Channel() chan dnsutils.DnsMessage {
 	return nil
 }
 
-func (c *DnstapRelay) Stop() {
+func (c *DnstapProxifier) Stop() {
 	c.LogInfo("stopping...")
 
 	// closing properly current connections if exists
@@ -139,22 +139,22 @@ func (c *DnstapRelay) Stop() {
 	close(c.done)
 }
 
-func (c *DnstapRelay) Listen() error {
+func (c *DnstapProxifier) Listen() error {
 	c.LogInfo("running in background...")
 
 	var err error
 	var listener net.Listener
-	addrlisten := c.config.Collectors.DnstapRelay.ListenIP + ":" + strconv.Itoa(c.config.Collectors.DnstapRelay.ListenPort)
+	addrlisten := c.config.Collectors.DnstapProxifier.ListenIP + ":" + strconv.Itoa(c.config.Collectors.DnstapProxifier.ListenPort)
 
 	if len(c.sockPath) > 0 {
 		_ = os.Remove(c.sockPath)
 	}
 
 	// listening with tls enabled ?
-	if c.config.Collectors.DnstapRelay.TlsSupport {
+	if c.config.Collectors.DnstapProxifier.TlsSupport {
 		c.LogInfo("tls support enabled")
 		var cer tls.Certificate
-		cer, err = tls.LoadX509KeyPair(c.config.Collectors.DnstapRelay.CertFile, c.config.Collectors.DnstapRelay.KeyFile)
+		cer, err = tls.LoadX509KeyPair(c.config.Collectors.DnstapProxifier.CertFile, c.config.Collectors.DnstapProxifier.KeyFile)
 		if err != nil {
 			c.logger.Fatal("loading certificate failed:", err)
 		}
@@ -166,7 +166,7 @@ func (c *DnstapRelay) Listen() error {
 		}
 
 		// update tls min version according to the user config
-		tlsConfig.MinVersion = dnsutils.TLS_VERSION[c.config.Collectors.DnstapRelay.TlsMinVersion]
+		tlsConfig.MinVersion = dnsutils.TLS_VERSION[c.config.Collectors.DnstapProxifier.TlsMinVersion]
 
 		if len(c.sockPath) > 0 {
 			listener, err = tls.Listen(dnsutils.SOCKET_UNIX, c.sockPath, tlsConfig)
@@ -191,7 +191,7 @@ func (c *DnstapRelay) Listen() error {
 	return nil
 }
 
-func (c *DnstapRelay) Run() {
+func (c *DnstapProxifier) Run() {
 	c.LogInfo("starting collector...")
 	if c.listen == nil {
 		if err := c.Listen(); err != nil {

collectors/dnstaprelay_test.go → collectors/dnstap_proxifier_test.go

@@ -14,13 +14,13 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
-func TestDnstapRelay_TcpRun(t *testing.T) {
+func TestDnstapProxifier_TcpSocket(t *testing.T) {
 	g := loggers.NewFakeLogger()
 
 	config := dnsutils.GetFakeConfig()
-	config.Collectors.DnstapRelay.ListenPort = 6100
+	config.Collectors.DnstapProxifier.ListenPort = 6100
 
-	c := NewDnstapRelay([]dnsutils.Worker{g}, config, logger.New(false), "test")
+	c := NewDnstapProxifier([]dnsutils.Worker{g}, config, logger.New(false), "test")
 	if err := c.Listen(); err != nil {
 		log.Fatal("collector dnstap relay tcp listening error: ", err)
 	}
@@ -69,17 +69,17 @@ func TestDnstapRelay_TcpRun(t *testing.T) {
 	}
 }
 
-func TestDnstapRelay_UnixRun(t *testing.T) {
+func TestDnstapProxifier_UnixSocket(t *testing.T) {
 	g := loggers.NewFakeLogger()
 	config := dnsutils.GetFakeConfig()
-	config.Collectors.DnstapRelay.SockPath = "/tmp/dnscollector_relay.sock"
-	c := NewDnstapRelay([]dnsutils.Worker{g}, config, logger.New(false), "test")
+	config.Collectors.DnstapProxifier.SockPath = "/tmp/dnscollector_relay.sock"
+	c := NewDnstapProxifier([]dnsutils.Worker{g}, config, logger.New(false), "test")
 	if err := c.Listen(); err != nil {
 		log.Fatal("collector dnstap replay unix listening  error: ", err)
 	}
 	go c.Run()
 
-	conn, err := net.Dial(dnsutils.SOCKET_UNIX, config.Collectors.DnstapRelay.SockPath)
+	conn, err := net.Dial(dnsutils.SOCKET_UNIX, config.Collectors.DnstapProxifier.SockPath)
 	if err != nil {
 		t.Error("could not connect to unix socket: ", err)
 	}