Fix some errors in documentations (dmachard#191)

Update README.md

README.md

@@ -18,24 +18,21 @@ DNS-collector also contains DNS parser with [`EDNS`](doc/dnsparser.md) support.
     - Protobuf [`PowerDNS`](doc/collectors.md#protobuf-powerdns) streams
     - [`Proxifier`](doc/collectors.md#dns-tap-proxifier) for DNSTap streams
 - *Live capture on a network interface*   
-    - [`AF_PACKET`](doc/collectors.md#dns-sniffer) socket with BPF filter
+    - [`AF_PACKET`](doc/collectors.md#live-capture-with-af_packet) socket with BPF filter
 - *Read text or binary files as input*
     - Read and tail on [`Plain text`](doc/collectors.md#tail) files
     - Ingest [`PCAP`](doc/collectors.md#file-ingestor) or [`DNSTap`](doc/collectors.md#file-ingestor) files by watching a directory
 
 **Loggers**:
-- *Redirect DNS logs to stdout or files in plain text or binary mode*
-    - Print directly to your [`Stdout`](doc/loggers.md#stdout) console
-    - Write to [`File`](doc/loggers.md#log-file) with several formats
-        - [Custom Text](doc/configuration.md#custom-text-format)
-        - [Json](doc/dnsjson.md)
-        - [Pcap](doc/loggers.md#log-file)
-        - [Dnstap](doc/loggers.md#log-file)
+
+- *Local storage of your DNS logs in plain [`Text`](doc/configuration.md#custom-text-format),  [`Json`](doc/dnsjson.md), [`Pcap`](doc/loggers.md#log-file) or [`Dnstap`](doc/loggers.md#log-file) formats:*
+    - [`Stdout`](doc/loggers.md#stdout) console
+    - [`File`](doc/loggers.md#log-file) with automatic rotation and compression
 - *Provide metrics and API*
     - [`Prometheus`](doc/loggers.md#prometheus) metrics and visualize-it with built-in [dashboards](doc/dashboards.md) for Grafana
     - [`Statsd`](doc/loggers.md#statsd-client) support
     - [`REST API`](doc/loggers.md#rest-api) with [swagger](https://generator.swagger.io/?url=https://raw.githubusercontent.com/dmachard/go-dnscollector/main/doc/swagger.yml) to search DNS domains
-- *Send to remote host with generic protocol*
+- *Send to remote host with generic transport protocol*
     - [`TCP`](doc/loggers.md#tcp-client)
     - [`Syslog`](doc/loggers.md#syslog)
     - [`DNSTap`](doc/loggers.md#dnstap-client) protobuf messages

config.yml

@@ -128,16 +128,12 @@ multiplexer:
 #   # private key server file
 #   key-file: ""
 
-# # live capture
+# # live capture with AF_PACKET
 # sniffer:
 #   # filter on source and destination port
 #   port: 53
 #   # if "" bind on all interfaces
 #   device: wlp2s0
-#   # drop all queries
-#   drop-queries: true
-#   # drop all replies
-#   drop-replies: true
 #   # The cache is used to compute latency between replies and queries
 #   cache-support: true
 #   # Ttl in second, max time to keep the query record in memory cache

doc/collectors.md

@@ -4,7 +4,7 @@
 - [DNStap Proxifier](#dns-tap-proxifier)
 - [Protobuf PowerDNS](#protobuf-powerdns)
 - [Tail](#tail)
-- [Live capture](#live-capture)
+- [Live capture with AF_PACKET](#live-capture-with-af_packet)
 - [File Ingestor](#file-ingestor)
 
 ## Collectors
@@ -72,7 +72,7 @@ dnstap-relay:
   key-file: ""
 ```
 
-### Live Capture
+### Live Capture with AF_PACKET
 
 Raw DNS packets sniffer. Setting `CAP_NET_RAW` capabilities on executables allows you to run these
 program without having to run-it with the root user:

doc/docker.md

@@ -1,7 +1,76 @@
 # DNS-collector - Docker
 
+## Docker run
+
 Docker run with a custom configuration:
 
 ```bash
 docker run -d dmachard/go-dnscollector -v $(pwd)/config.yml:/etc/dnscollector/config.yml
 ```
+## Docker-compose
+
+Example with docker-compose
+
+```
+version: "3.8"
+
+services:
+  dnscollector:
+    image: dmachard/go-dnscollector:v0.25.0
+    environment:
+      - "TZ=Europe/Paris"
+    volumes:
+      - ${APP_CONFIG}/dnscollector/config.yml:/etc/dnscollector/config.yml
+      - ${COLLECTOR_DATA}/:/var/dnscollector/
+    ports:
+      - "8080:8080/tcp"
+      - "8081:8081/tcp"
+      - "6000:6000/tcp"
+```
+
+DNS-collector configuration:
+
+```
+global:
+  trace:
+    verbose: true
+    log-malformed: true
+
+multiplexer:
+  collectors:
+    - name: tap
+      powerdns:
+        listen-ip: 0.0.0.0
+        listen-port: 6000
+      transforms:
+        normalize:
+          lowercase-qname: true
+        suspicious:
+          enable: true
+        public-suffix:
+          add-tld: true
+
+  loggers:
+    - name: console
+      stdout:
+        mode: text
+
+    - name: json
+      logfile:
+        file-path:  /var/dnscollector/dnstap.log 
+        mode: text
+
+    - name: api
+      restapi:
+        listen-ip: 0.0.0.0
+        listen-port: 8080
+
+    - name: prom
+      prometheus:
+        listen-ip: 0.0.0.0
+        listen-port: 8081
+ 
+  routes:
+    - from: [ tap ]
+      to: [ console, json, api, prom ]
+```

doc/loggers.md

@@ -122,9 +122,9 @@ Enable this logger if you want to log your DNS traffic to a file in plain text m
 * custom text format
 
 For config examples, take a look to the following links:
-- [text](https://github.com/dmachard/go-dns-collector/blob/main/example-config/use-case-7.yml).
-- [dnstap](https://github.com/dmachard/go-dns-collector/blob/main/example-config/use-case-13.yml).
-- [pcap](https://github.com/dmachard/go-dns-collector/blob/main/example-config/use-case-13.yml).
+- [`text`](https://github.com/dmachard/go-dns-collector/blob/main/example-config/use-case-7.yml)
+- [`dnstap`](https://github.com/dmachard/go-dns-collector/blob/main/example-config/use-case-13.yml)
+- [`pcap`](https://github.com/dmachard/go-dns-collector/blob/main/example-config/use-case-1.yml)
 
 Options:
 - `file-path`: (string) output logfile name

doc/overview.drawio

@@ -1 +1 @@
-<mxfile host="app.diagrams.net" modified="2022-11-13T06:31:27.699Z" agent="5.0 (X11)" etag="XESKpodHZSGLUGk0bACG" version="20.0.2" type="device"><diagram id="ufO1G3qEnvI_ADMoatvI" name="Page-1">7Vxte5o6GP41XtfOB7mAhAAftbbdetquXdut6zcqUdmQOMSq+/UnKCgk4cUKottxH0aehLc79/NK0hY4Gy8ufWsyuiE2dluqbC9aoNdSVUUxNPpfKFmuJUhV1oKh79jRoK3gwfmNI6EcSWeOjaepgQEhbuBM0sI+8TzcD1Iyy/fJPD1sQNz0XSfWEHOCh77l8tJvjh2M1lJDk7fyj9gZjuI7K3LUM7biwZFgOrJsMk+IwHkLnPmEBOuj8eIMuyF4MS7r8y4yejcP5mMvKHNC97f2OFanr5Pnb7eeYXUuf5hn7egqb5Y7i144ethgGSPgk5ln4/Aicgt05yMnwA8Tqx/2zumcU9koGLu0pdDDaeCTn/iMuMSnEo94dFh34LhuLGqpYKCF/0I58YKEfP2jcv7N4sfEfoAXCVH0ppeYjHHgL+mQqFeFEepLpj3fTqIWc2yUmEDVjIRWRJzh5tpbbOlBBO8OUINdoFaKoWZQtS1sDPpZU2D5/Ui35IoARmYaYE2VNA5iJb5wCmKkx0MrBxkKQEYuvXH3lR4MwwOKjEstBfGncRe906aXmxOKR7Ajx2PMXWfo0WafooypvBui61DT0ok6xo5th7cRznRa7SqYMSDDEjMGRTpRl0po9arEYDBQ+4dTCaQwNkfTJVWgE0AEcY06gYp14poMh/hvUwikAUnXykzZQZVC56bL9qYBhaB6x1yFHzCQxGBo8K5W35iaJISwLghNDkJK8AuHAneUGCIIJWCkQATUHsjJH+IxFZASalJsQysHVeFjxRt6Yac/PRlUhdQUwahKcl0wxi+SgAvbNOWImsQPRmRIPMs930oZ27cdc03IJILxBw6CZeTQrFlA0iBTGP3lc3T+qvE9bITGLmr3Fsne3jJqZcXu6VmjPWj12/TEyZKaN5mB5Q9xUGwLQ4Byp9zHrhU4b+mcTTR10al3xKGPsjVijOcGMmOZpmTm93F01pYAHd+3lolhk3DAtPx94mRjy6f1FTPO1pin1BmOruHknnJF0w1YpZirteHrHM2Vl4/98fkTev7aU2Ab6sfE3CLi8jS0reloE0mGjTsroHGIt5KospLH1DUB1qI2WL58QWQx+XZz5Sxu7N7yqtMWMFoIIlBqofSuTERGmktQh0nK7Do+zdx3EM55/DS66i2ca3mp6R332RrOFwI3HgeotvMWx6eB5biJuDXRc5xuiQ+YgK5Jppbw9XwAKo6fQCU5g/b55sW4QTIkT98/Xc9e3ixLF9SFJEk6FUAhOmQE+v3xdnT907j/0fHm3atfZ/9ePk3inOuwppJLfzESpL9hrUg3X+USRrKUORS+Pyjp4M2jsIaaqacJJDOl1ILxYCc/jnRTAmbil743y8nqvHreTOUb2ZtZWHF38YKm7Rm2ljvnC5kFjjfc1Bp8dmyV1pqlvtHH2ZWfCowOZGMxqIvKn7IguTD0mmwOPKbwrDCxwAsneI4vQo8TUR1tbU8KG9kxHVsE17BhQ5HJM9RXsEpPStm0vGQkadPyNKphmwZlJsoosGnc+KjAX1WEJ4RKa4Sy72BeM641Lyo+FRqyqbWp5dMwf/zeNFzMexe3S3LRbxv3ny9n7c7iK27GcjZDqbxo/1QoJTO5qFFk2Zjx6i7RGmC+JG88f0FtiH8MxFxIqS3QE5K8EVtbRfkwK8TQi0KMcgohqtKovEI0WZ9UEOObDVVS9G3BQCnFyGw9y+SsqutssUJR2aJDzbxtpOpYa9m7kH6C7DkvFKidfjqSTJkmqhCpyDAVGTAu2mApUVW9XGMtt5JPV3a8vFN9nb6mypbmUH1sFxfZW3w6vk9iXFD0zuVp+XRYAZCFLlFxS2TEhiAhBhUkxEIsRYvbuDLFHZljv3f7ULquYXtT25kGh6lrHGj6VH76DNDs5O20Nuh4kOQVAQC+Gn1QJFUOySfvNYTuJADlqQlgGP00iym/SOckwASs5waSbDSLJF9/7jqnQs00mlBu2GTGCeVxVIAbXVlSGGILqmh5pZH6MzxD0tOLlaCpSIqW+EzE0KaqGBvsVk0pGF8YY9OEEuz0njUH3MaJWnLOLULUvDHnV2zsvWygsTAD6g2bc8DnLw+O93PP5Zb7rIIqj6fOrhfStKL1rAfFFjZS8q9vTZr4JVXeyeUu7W24sM+SBkQsyXJFBeP3/laUC2pCK+/88KojPDtJ1VSNI1PNZj4KVxPFlliA8M4odq9lVkei4JrOKGxBrMmNl5VWpoLzZyPx3Xb+coe0cCcB97llU44wN9uIqo9ZP9/rY/VLu6N+nP8av5A7e941BRWdVJ0yoTro1yzcX7zaatuersjfoQPMyWJdsIy62RJmOHyf6zz6ljcdEH+c3FK2vmh+rfSo3uLDne+8WX3KcDpNMo1k/8kt+pZ5P9awua4zmeJi1/CO9W2cRxCYmEwnARST0TxTUM1HAr9QxZJaIemP4vv13os03utnhPa7MqeRvaXh4C7CZLIw9q8aFI3Xc5atFboIyO7q3MlF6JlrejWjLv+QtxnlFFbvC3bp0VyRszTCXXqVbH8Q2hpRef+oXNO7HeypOCBuw5EBN1FWUx4orq//gbQ4qrf4QMlG897V5oG/LvLiiB+vXmyM9QXLKJrnyx9vDNlo/CiM4Z+bhJ4sLRo3FQWbyf7nxMHdR62JO21u/37dOnnZ/hVAcP4f</diagram></mxfile>
+<mxfile host="app.diagrams.net" modified="2022-12-10T10:09:57.374Z" agent="5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36" etag="tIZxdix1-g7IzgqjC94A" version="20.6.2" type="device"><diagram id="ufO1G3qEnvI_ADMoatvI" name="Page-1">7Vxte5o6GP41XtfOB7mAhAAftbbdetquXdut6zcqUdmQOMCq+/UnKCgkEbCC6HbcPpAn4e3O/byStAXOxvNL35qMboiN3ZYq2/MW6LVUVYGq2or+y/ZiJdENtBIMfceOB20ED85vHAvlWDp1bBxkBoaEuKEzyQr7xPNwP8zILN8ns+ywAXGzd51YQ8wJHvqWy0u/OXY4WkkNTd7IP2JnOErurMhxz9hKBseCYGTZZJYSgfMWOPMJCVdH4/kZdiPwElxW511s6V0/mI+9sMwJ3d/a41gNXifP3249w+pc/jDP2vFV3ix3Gr9w/LDhIkHAJ1PPxtFF5BbozkZOiB8mVj/qndE5p7JROHZpS6GHQeiTn/iMuMSnEo94dFh34LhuImqpYKBF/yI58cKUfPWjcv7NksfEfojnKVH8ppeYjHHoL+iQuFeFMeoLpj3bTKKWcGyUmkDVjIVWTJzh+tobbOlBDO8OUINdoFaKoWZQtS1sDPrbpsDy+7FuyRUBjMwswJoqaRzESnLhDMRIT4ZWDjIUgIxceuPuKz0YRgcUGZdaCuIHSRe907qXmxOKR7gjxxPMXWfo0WafooypvBuh61DT0ok7xo5tR7cRznRW7SqYMSDDEjMGRTpRl0po9arEYDBQ+4dTCaQwNkfTJVWgE0AEcY06gYp14poMh/hvUwikAUnXykzZQZVC56bL9oKQQlC9Y67CDxhIYjA0eFerr01NGkJYF4QmByEl+IVDgTtKDBGEEjAyIAJqD+T0D/GYCkgJNSmxoZWDqvCx4g29sNMPTgZVITVFMKqSXBeMyYuk4MI2TTniJvHDERkSz3LPN1LG9m3GXBMyiWH8gcNwETs0axqSLMgURn/xHJ+/bHyPGpGxi9u9ebq3t4hb22L37KzRHrT8rXuSZEnNm8zQ8oc4LLaFEUC5U+5j1wqdt2zOJpq6+NQ74tBH2RgxxnMDmbFMAZn6fRyftSFAx/etRWrYJBoQlL9Pkmxs+LS64pazNeYpdYajKzi5p1zSdA1WKeZqbfg6QzPl5WN/fP6Enr/2FNiG+jExt4i4PA1tKxitI8mocWeFNA7xlhJVVvKYuiLAStQGi5cviMwn326unPmN3VtcddoCRgtBBEotlN6VicjIcgnqME2ZXcdnmfsOwjmPn0ZXvblzLS80veM+W8PZXODGkwDVdt6S+DS0HDcVt6Z6jtMt8QET0DXJ1FK+ng9AxfETqCRn0D7fvBg3SIbk6fun6+nLm2XpgrqQJEmnAihEh4xAvz/ejq5/Gvc/Ot6se/Xr7N/Lp0mScx3WVHLpL0aC9DeqFenmq1zCSJYyh8L3ByUdvHkU1lAz9SyBZKaUWjAe7OTHkW5KwEz9svdmOVmdV8+bqXwjezONKu4untO0fYut5c75Qqah4w3XtQafHVultWapb/Tx9spPBUYHsrEY1EXlT1mQXBh6TTYHHlN4VphY4LkTPicXocepqI62NidFje0xHVsE17BhQ5HJM9RXsExPStm0vGQkbdPyNKphmwZlJsoosGnc+LjAX1WEJ4RKa4Sy72BeM641Lyo+FRqyqbWp5dMwf/zeNJzPehe3C3LRbxv3ny+n7c78K27GcjZDqbxo/1QoJTO5qFFk2Zjx6i7RGmC+JK89f0FtiH8MxFxIqS3QE5K8EVtbRflwW4ihF4UY5RRCVKVReYVosj6pIMY3G6qk6JuCgVKKkdv1bCtnVV1nixWKyhYdauZtI1XHWsvehfQTZM95oUDt9NORZMo0UYVIRYapyIBx0QZLiarq5RpruZV8urLj5Z3q6/Q1VbY0h+pju7jI3uLT8X0S44Kidy5Py6fDCoAsdKmKWyojNgQJMaggIRZiKVrcxpUp7sgM+73bh9J1DdsLbCcID1PXOND0qfz0GaDZydtpbdDxIMkrAgB8NfqgSKockk/eawTdSQDKUxPAKPppFlN+kc5JgAlYzw0k2WgWSb7+3HVOhZpZNKHcsMlMEsrjqAA3urKkMMQWVNHySiP1Z3iGpGcXK0FTkRQt9ZmIoU1VMTbYrZpSML4wxqYJJdjpPWsOuI0TteScW4SoeWPOr9jYe9lAY2EG1Bs254DPXx4c7+eeyy33WQVVHk+dXS+kaUXrWQ+KLWyk5F/fmjTxS6q8k8td2ttwYZ8lDYhZss0VFYzf+1tRLqgprbzzo6uO8PQkVVM1jkw1m/koXE0UW2IBwjuj2L2WWR2Jgms6o7AFsSY3XlZaWxWcPxuJ77bzlzukRTsJuM8t63KEud5GVH3M+vleH6tf2h314+zX+IXc2bOuKajoZOqUKdVBv6bR/uLlVtt2sCR/hw4wJ/NVwTLuZkuY0fB9rvPoW14wIP44vaVsddH8WulRvcWHO995s/qU4XSaZBrJ/pNb9C3zfqxhc11nEuBi13Do9W1AMRnNMwXVfCTwC1UsqRWS/ii+X++9SOO9fqZSH5G3g6FpF2EyWRj7Vw2Kxus5y9YKXQRkd3Xu5CL0rWt6NaMu/5A3laewel+wS4/mipylEe7Sq2T7g9DWiMr7R+Wa3u1gT8UBcRuODLiOspryQEl9/Q+kxVG9xQdKNpr3LjcP/HWRF0f8ZPViY6wvWEbRPF/+eGPIRuNHYQz/3CT0ZGnRuKko2Ez2PycO7j5qTdxpc/P361bJy+avAILz/wA=</diagram></mxfile>

doc/transformers.md

@@ -99,19 +99,15 @@ Example:
 
 ```json
 {
-  ...
   "geo": {
     "city": "-",
     "continent": "-",
     "country-isocode": "-"
   },
   "network": {
-    ...
     "as-number": 1234,
     "as-owner": "Orange",
   },
-  ...
-}
 ```
 
 ### DNS filtering

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/dmachard/go-logger v0.3.0
 	github.com/dmachard/go-powerdns-protobuf v0.0.4
 	github.com/dmachard/go-topmap v0.5.0
+	github.com/farsightsec/golang-framestream v0.3.0
 	github.com/fsnotify/fsnotify v1.6.0
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/gopacket v1.1.19
@@ -30,7 +31,6 @@ require (
 )
 
 require (
-	github.com/farsightsec/golang-framestream v0.3.0 // indirect
 	go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect
 	go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 // indirect
 )