WIP: TZSP support (dmachard#172)

* Full support tzsp encapsulation protocol (used by Mikrotik devices for sending of data, can be used to capture DNS packets) * remove debug prints from tzsp processor * skip processing of packets that are too short (less than 12 byte only for example) * gofmt'd the tzsp processor code * check for err in address resolution * gofmt'd config.go * make "0.0.0.0" a constant * amend documentation * fix * remove tap from collector list * only build tzsp_processor.go on Linux * tzsp_processor implementation for darwin; Only syscall and usage of timestamp is different * doc/collectors: change highlighting to routeros * collectors/tzsp_processor_darwin.go: try to use ReadMsgUDP instead of ReadMsgUDPAddrPort * collectors/tzsp_processor_darwin.go: gofmt * try * test * remove incomplete macos implementation * Bump github.com/cilium/ebpf from 0.9.3 to 0.10.0 (dmachard#198) Bumps [github.com/cilium/ebpf](https://github.com/cilium/ebpf) from 0.9.3 to 0.10.0. - [Release notes](https://github.com/cilium/ebpf/releases) - [Commits](cilium/ebpf@v0.9.3...v0.10.0) --- updated-dependencies: - dependency-name: github.com/cilium/ebpf dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <[email protected]> Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Optimization - breaking changes in JSON format (dmachard#218) * fix dns parser: ignore too short decode error if reply is truncated (dmachard#221) * bypass macos and windows * fix readme.md --------- Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: Denis MACHARD <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
flowintel · Feb 23, 2023 · 6810e63 · 6810e63
1 parent 2b38eda
commit 6810e63
Show file tree

Hide file tree

Showing 13 changed files with 569 additions and 8 deletions.
diff --git a/README.md b/README.md
@@ -18,12 +18,13 @@ Additionally, DNS-collector also contains DNS parser with [`EDNS`](doc/dnsparser
     - Protobuf [`DNStap`](doc/collectors.md#dns-tap) with `tls`, `tcp` or `unix` support
     - Protobuf [`PowerDNS`](doc/collectors.md#protobuf-powerdns) streams with full [`metadata`](doc/powerdns.md) support 
     - [`Proxifier`](doc/collectors.md#dns-tap-proxifier) for DNSTap streams
-- *Live capture on a network interface*   
+- *Live capture on a network interface*
     - [`AF_PACKET`](doc/collectors.md#live-capture-with-af_packet) socket with BPF filter
     - [`eBPF XDP`](doc/collectors.md#live-capture-with-ebpf-xdp) ingress traffic
 - *Read text or binary files as input*
     - Read and tail on [`Plain text`](doc/collectors.md#tail) files
     - Ingest [`PCAP`](doc/collectors.md#file-ingestor) or [`DNSTap`](doc/collectors.md#file-ingestor) files by watching a directory
+- Listen for [`TZSP`](doc/collectors.md#tzsp) packets containg DNS packets
 
 **Loggers**:
 
@@ -86,14 +87,13 @@ See the full [configuration guide](doc/configuration.md) for more details.
 
 You will find below some examples of configuration to manage your DNS logs.
 
-
-- Capture DNS traffic with incoming DNSTap streams
+- Capture DNS traffic from incoming DNSTap streams
     - [x] [Read from UNIX DNSTap socket and forward it to TLS stream](example-config/use-case-5.yml)
     - [x] [Transform DNSTap as input to JSON format as output](example-config/use-case-3.yml)
     - [x] [Relays DNSTap stream to multiple remote destination without decoding](example-config/use-case-12.yml)
     - [x] [Aggregate several DNSTap stream and forward it to the same file](example-config/use-case-7.yml)
 
-- Capture DNS traffic with PowerDNS
+- Capture DNS traffic from PowerDNS products
     - [x] [Capture multiple PowerDNS streams](example-config/use-case-8.yml)
 
 - Observe your DNS traffic from logs
@@ -113,6 +113,9 @@ You will find below some examples of configuration to manage your DNS logs.
 - Capture DNS traffic from PCAP files
     - [x] [Capture DNSTap stream and backup-it to text and pcap files](example-config/use-case-1.yml)
     - [x] [Watch for PCAP files as input and JSON as output](example-config/use-case-15.yml)
+
+- Capture DNS traffic from Mikrotik device
+    - [x] [Capture TZSP packets containing DNS packets and process them as json](example-config/use-case-17.yml)
 
 ## Contributing
 

diff --git a/collectors/tzsp.go b/collectors/tzsp.go
@@ -0,0 +1,276 @@
+//go:build linux
+// +build linux
+
+// Written by Noel Kuntze <noel.kuntze {@@@@@} thermi.consulting>
+
+package collectors
+
+import (
+	"encoding/binary"
+	"fmt"
+	"net"
+	"syscall"
+
+	"github.com/dmachard/go-dnscollector/dnsutils"
+	"github.com/dmachard/go-logger"
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/rs/tzsp"
+)
+
+type TzspSniffer struct {
+	done        chan bool
+	exit        chan bool
+	listen      net.UDPConn
+	loggers     []dnsutils.Worker
+	config      *dnsutils.Config
+	logger      *logger.Logger
+	name        string
+	identity    string
+	port        int
+	ip          string
+	dropQueries bool
+	dropReplies bool
+}
+
+func NewTzsp(loggers []dnsutils.Worker, config *dnsutils.Config, logger *logger.Logger, name string) *TzspSniffer {
+	logger.Info("[%s] tzsp collector - enabled", name)
+	s := &TzspSniffer{
+		done:    make(chan bool),
+		exit:    make(chan bool),
+		config:  config,
+		loggers: loggers,
+		logger:  logger,
+		name:    name,
+	}
+	s.ReadConfig()
+	return s
+}
+
+func (c *TzspSniffer) GetName() string { return c.name }
+
+func (c *TzspSniffer) SetLoggers(loggers []dnsutils.Worker) {
+	c.loggers = loggers
+}
+
+func (c *TzspSniffer) Loggers() []chan dnsutils.DnsMessage {
+	channels := []chan dnsutils.DnsMessage{}
+	for _, p := range c.loggers {
+		channels = append(channels, p.Channel())
+	}
+	return channels
+}
+
+func (c *TzspSniffer) LogInfo(msg string, v ...interface{}) {
+	c.logger.Info("["+c.name+"] tzsp collector - "+msg, v...)
+}
+
+func (c *TzspSniffer) LogError(msg string, v ...interface{}) {
+	c.logger.Error("["+c.name+"] tzsp collector - "+msg, v...)
+}
+
+func (c *TzspSniffer) ReadConfig() {
+
+	c.port = c.config.Collectors.Tzsp.ListenPort
+	c.ip = c.config.Collectors.Tzsp.ListenIp
+	c.identity = c.config.GetServerIdentity()
+	c.dropQueries = c.config.Collectors.Tzsp.DropQueries
+	c.dropReplies = c.config.Collectors.Tzsp.DropReplies
+	// TODO: Implement
+}
+
+func (c *TzspSniffer) Listen() error {
+	c.logger.Info("running in background...")
+
+	ServerAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", c.ip, c.port))
+	if err != nil {
+		return err
+	}
+
+	ServerConn, err := net.ListenUDP("udp", ServerAddr)
+	if err != nil {
+		return err
+	}
+	file, err := ServerConn.File()
+
+	if err != nil {
+		return err
+	}
+
+	err = syscall.SetsockoptInt(int(file.Fd()), syscall.SOL_SOCKET, syscall.SO_TIMESTAMPNS, 1)
+
+	if err != nil {
+		return err
+	}
+	c.LogInfo("is listening on %s", ServerConn.LocalAddr())
+	c.listen = *ServerConn
+	return nil
+}
+
+func (c *TzspSniffer) Channel() chan dnsutils.DnsMessage {
+	return nil
+}
+
+func (c *TzspSniffer) Stop() {
+	c.LogInfo("stopping...")
+
+	// Finally close the listener to unblock accept
+	c.exit <- true
+
+	// read done channel and block until run is terminated
+	<-c.done
+	close(c.done)
+}
+
+func (c *TzspSniffer) Run() {
+	c.logger.Info("starting collector...")
+
+	if err := c.Listen(); err != nil {
+		c.logger.Fatal("collector tzsp listening failed: ", err)
+	}
+
+	dnsProcessor := NewDnsProcessor(c.config, c.logger, c.name)
+	dnsProcessor.cacheSupport = c.config.Collectors.Tzsp.CacheSupport
+	dnsProcessor.queryTimeout = c.config.Collectors.Tzsp.QueryTimeout
+
+	go dnsProcessor.Run(c.Loggers())
+
+	go func() {
+		buf := make([]byte, 65536)
+		oob := make([]byte, 100)
+		for {
+			//flags, from
+			bufN, oobn, _, _, err := c.listen.ReadMsgUDPAddrPort(buf, oob)
+			if err != nil {
+				panic(err)
+			}
+			if bufN == 0 {
+				panic("buf empty")
+			}
+			if bufN > len(buf) {
+				panic("buf overflow")
+			}
+			if oobn == 0 {
+				panic("oob missing")
+			}
+			c.LogInfo("Packet received")
+			scms, err := syscall.ParseSocketControlMessage(oob[:oobn])
+			if err != nil {
+				panic(err)
+			}
+			if len(scms) != 1 {
+				c.LogInfo("len(scms) != 1")
+				continue
+			}
+			scm := scms[0]
+			if scm.Header.Type != syscall.SCM_TIMESTAMPNS {
+				panic("scm timestampns missing")
+			}
+			tsec := binary.LittleEndian.Uint32(scm.Data[:4])
+			nsec := binary.LittleEndian.Uint32(scm.Data[8:12])
+
+			// copy packet data from buffer
+			pkt := make([]byte, bufN)
+			copy(pkt, buf[:bufN])
+
+			tzsp_packet, err := tzsp.Parse(pkt)
+
+			if err != nil {
+				c.LogError("Failed to parse packet: ", err)
+				continue
+			}
+
+			var eth layers.Ethernet
+			var ip4 layers.IPv4
+			var ip6 layers.IPv6
+			var tcp layers.TCP
+			var udp layers.UDP
+			parser := gopacket.NewDecodingLayerParser(layers.LayerTypeEthernet, &eth, &ip4, &ip6, &tcp, &udp)
+			decodedLayers := make([]gopacket.LayerType, 0, 4)
+
+			// decode-it
+			parser.DecodeLayers(tzsp_packet.Data, &decodedLayers)
+
+			dm := dnsutils.DnsMessage{}
+			dm.Init()
+
+			ignore_packet := false
+			for _, layertyp := range decodedLayers {
+				switch layertyp {
+				case layers.LayerTypeIPv4:
+					dm.NetworkInfo.Family = dnsutils.PROTO_IPV4
+					dm.NetworkInfo.QueryIp = ip4.SrcIP.String()
+					dm.NetworkInfo.ResponseIp = ip4.DstIP.String()
+
+				case layers.LayerTypeIPv6:
+					dm.NetworkInfo.QueryIp = ip6.SrcIP.String()
+					dm.NetworkInfo.ResponseIp = ip6.DstIP.String()
+					dm.NetworkInfo.Family = dnsutils.PROTO_IPV6
+
+				case layers.LayerTypeUDP:
+					dm.NetworkInfo.QueryPort = fmt.Sprint(int(udp.SrcPort))
+					dm.NetworkInfo.ResponsePort = fmt.Sprint(int(udp.DstPort))
+					dm.DNS.Payload = udp.Payload
+					dm.DNS.Length = len(udp.Payload)
+					dm.NetworkInfo.Protocol = dnsutils.PROTO_UDP
+
+				case layers.LayerTypeTCP:
+					// ignore SYN/ACK packet
+					// Note: disabled because SYN/SYN+Ack might contain data if TCP Fast open is used
+					// if !tcp.PSH {
+					// 	ignore_packet = true
+					// 	continue
+					// }
+					if len(tcp.Payload) < 12 {
+						// packet way too short; 12 byte is the minimum size a DNS packet (header only,
+						// no questions, answers, authorities, or additional RRs)
+						continue
+					}
+					dnsLengthField := binary.BigEndian.Uint16(tcp.Payload[0:2])
+					if len(tcp.Payload) < int(dnsLengthField) {
+						ignore_packet = true
+						continue
+					}
+
+					dm.NetworkInfo.QueryPort = fmt.Sprint(int(tcp.SrcPort))
+					dm.NetworkInfo.ResponsePort = fmt.Sprint(int(tcp.DstPort))
+					dm.DNS.Payload = tcp.Payload[2:]
+					dm.DNS.Length = len(tcp.Payload[2:])
+					dm.NetworkInfo.Protocol = dnsutils.PROTO_TCP
+				}
+			}
+
+			if !ignore_packet {
+				dm.DnsTap.Identity = c.identity
+
+				// set timestamp
+				dm.DnsTap.TimeSec = int(tsec)
+				dm.DnsTap.TimeNsec = int(nsec)
+
+				// just decode QR
+				if len(dm.DNS.Payload) < 4 {
+					continue
+				}
+				qr := binary.BigEndian.Uint16(dm.DNS.Payload[2:4]) >> 15
+
+				// is query ?
+				if int(qr) == 0 && !c.dropQueries {
+					dnsProcessor.GetChannel() <- dm
+				}
+
+				// is reply ?
+				if int(qr) == 1 && !c.dropReplies {
+					dnsProcessor.GetChannel() <- dm
+				}
+			}
+		}
+	}()
+
+	<-c.exit
+
+	// stop dns processor
+	dnsProcessor.Stop()
+
+	c.LogInfo("run terminated")
+	c.done <- true
+}
diff --git a/collectors/tzsp_darwin.go b/collectors/tzsp_darwin.go
@@ -0,0 +1,78 @@
+//go:build darwin
+// +build darwin
+
+package collectors
+
+import (
+	"github.com/dmachard/go-dnscollector/dnsutils"
+	"github.com/dmachard/go-logger"
+)
+
+type TzspSniffer struct {
+	done    chan bool
+	exit    chan bool
+	loggers []dnsutils.Worker
+	config  *dnsutils.Config
+	logger  *logger.Logger
+	name    string
+}
+
+// workaround for macos, not yet supported
+func NewTzsp(loggers []dnsutils.Worker, config *dnsutils.Config, logger *logger.Logger, name string) *AfpacketSniffer {
+	logger.Info("[%s] tzsp collector - enabled", name)
+	s := &AfpacketSniffer{
+		done:    make(chan bool),
+		exit:    make(chan bool),
+		config:  config,
+		loggers: loggers,
+		logger:  logger,
+		name:    name,
+	}
+	s.ReadConfig()
+	return s
+}
+
+func (c *TzspSniffer) GetName() string { return c.name }
+
+func (c *TzspSniffer) SetLoggers(loggers []dnsutils.Worker) {
+	c.loggers = loggers
+}
+
+func (c *TzspSniffer) LogInfo(msg string, v ...interface{}) {
+	c.logger.Info("["+c.name+"] tzsp collector - "+msg, v...)
+}
+
+func (c *TzspSniffer) LogError(msg string, v ...interface{}) {
+	c.logger.Error("["+c.name+"] tzsp collector - "+msg, v...)
+}
+
+func (c *TzspSniffer) Loggers() []chan dnsutils.DnsMessage {
+	channels := []chan dnsutils.DnsMessage{}
+	for _, p := range c.loggers {
+		channels = append(channels, p.Channel())
+	}
+	return channels
+}
+
+func (c *TzspSniffer) ReadConfig() {
+}
+
+func (c *TzspSniffer) Channel() chan dnsutils.DnsMessage {
+	return nil
+}
+
+func (c *TzspSniffer) Stop() {
+	c.LogInfo("stopping...")
+
+	// exit to close properly
+	c.exit <- true
+
+	// read done channel and block until run is terminated
+	<-c.done
+	close(c.done)
+}
+
+func (c *TzspSniffer) Run() {
+	c.LogInfo("run terminated")
+	c.done <- true
+}