feat(compression): Make the compression type configurable, default: zstd

Author: rnishtala-sumo

.changelog/3704.added.txt

@@ -0,0 +1 @@
+feat(compression): Make the compression type configurable, default: zstd

deploy/helm/sumologic/conf/events/otelcol/config.yaml

@@ -2,6 +2,7 @@ exporters:
   sumologic:
     client: {{ include "sumologic.sumo_client" . }}
     endpoint: {{ include "sumologic.events.exporter.endpoint" . }}
+    compression: {{ .Values.metadata.compression }}
     log_format: {{ include "sumologic.events.exporter.format" . }}
     sending_queue:
       enabled: true
@@ -18,6 +19,7 @@ exporters:
   sumologic/sumologic-mock:
     client: {{ include "sumologic.sumo_client" . }}
     endpoint: {{ include "sumologic-mock.receiver-endpoint" . }}
+    compression: {{ .Values.metadata.compression }}
     log_format: {{ include "sumologic.events.exporter.format" . }}
     sending_queue:
       enabled: true

deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml

@@ -1,6 +1,7 @@
 exporters:
   otlphttp:
     endpoint: http://${LOGS_METADATA_SVC}.${NAMESPACE}.svc.{{ .Values.sumologic.clusterDNSDomain }}.:4318
+    compression: {{ .Values.metadata.compression }}
     sending_queue:
       queue_size: 10
     # this improves load balancing at the cost of more network traffic

deploy/helm/sumologic/conf/logs/otelcol/config.yaml

@@ -8,6 +8,9 @@ exporters:
 {{ if .Values.sumologic.logs.container.enabled }}
   sumologic/containers:
     endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}
+    compression: {{ .Values.metadata.compression }}
+    json_logs:
+      add_timestamp: false
     log_format: {{ include "logs.otelcol.container.exporter.format" . }}
     sending_queue:
       enabled: true
@@ -20,6 +23,9 @@ exporters:
 {{- if eq (include "sumologic-mock.forward-logs-metadata" .) "true" }}
   sumologic/sumologic-mock-containers:
     endpoint: {{ include "sumologic-mock.receiver-endpoint" . }}
+    compression: {{ .Values.metadata.compression }}
+    json_logs:
+      add_timestamp: false
     log_format: {{ include "logs.otelcol.container.exporter.format" . }}
     sending_queue:
       enabled: true
@@ -35,6 +41,9 @@ exporters:
 {{ if .Values.sumologic.logs.systemd.enabled }}
   sumologic/systemd:
     endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}
+    compression: {{ .Values.metadata.compression }}
+    json_logs:
+      add_timestamp: false
     log_format: json
     sending_queue:
       enabled: true
@@ -47,6 +56,9 @@ exporters:
 {{- if eq (include "sumologic-mock.forward-logs-metadata" .) "true" }}
   sumologic/sumologic-mock-systemd:
     endpoint: {{ include "sumologic-mock.receiver-endpoint" . }}
+    compression: {{ .Values.metadata.compression }}
+    json_logs:
+      add_timestamp: false
     log_format: json
     sending_queue:
       enabled: true
@@ -63,6 +75,7 @@ exporters:
   sumologic:
     client: {{ include "sumologic.sumo_client" . }}
     endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_LOGS_SOURCE}
+    compression: {{ .Values.metadata.compression }}
     log_format: otlp
     sending_queue:
       enabled: true
@@ -76,6 +89,7 @@ exporters:
   sumologic/sumologic-mock:
     client: {{ include "sumologic.sumo_client" . }}
     endpoint: {{ include "sumologic-mock.receiver-endpoint" . }}
+    compression: {{ .Values.metadata.compression }}
     log_format: otlp
     sending_queue:
       enabled: true

deploy/helm/sumologic/conf/metrics/otelcol/exporters.yaml

@@ -8,6 +8,7 @@ sumologic/sumologic-mock-default:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: {{ include "metrics.otelcol.exporter.format" . }}
   endpoint: {{ include "sumologic-mock.receiver-endpoint" . }}
+  compression: {{ .Values.metadata.compression }}
   ## Sumo doesn't yet natively support OTLP Histograms
   decompose_otlp_histograms: true
   ## Configuration for sending queue
@@ -29,6 +30,7 @@ sumologic/sumologic-mock-http:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: prometheus
   endpoint: {{ include "sumologic-mock.receiver-endpoint" . }}
+  compression: {{ .Values.metadata.compression }}
   sending_queue:
     enabled: true
 {{- if .Values.metadata.persistence.enabled }}
@@ -47,6 +49,7 @@ sumologic/default:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: {{ include "metrics.otelcol.exporter.format" . }}
   endpoint: {{ include "metrics.otelcol.exporter.endpoint" . }}
+  compression: {{ .Values.metadata.compression }}
   ## Sumo doesn't yet natively support OTLP Histograms
   decompose_otlp_histograms: true
   ## Configuration for sending queue
@@ -68,6 +71,7 @@ sumologic/apiserver:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: prometheus
   endpoint: ${SUMO_ENDPOINT_APISERVER_METRICS_SOURCE}
+  compression: {{ .Values.metadata.compression }}
   sending_queue:
     enabled: true
 {{- if .Values.metadata.persistence.enabled }}
@@ -82,6 +86,7 @@ sumologic/control_plane:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: prometheus
   endpoint: ${SUMO_ENDPOINT_CONTROL_PLANE_METRICS_SOURCE}
+  compression: {{ .Values.metadata.compression }}
   sending_queue:
     enabled: true
 {{- if .Values.metadata.persistence.enabled }}
@@ -96,6 +101,7 @@ sumologic/controller:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: prometheus
   endpoint: ${SUMO_ENDPOINT_CONTROLLER_METRICS_SOURCE}
+  compression: {{ .Values.metadata.compression }}
   sending_queue:
     enabled: true
 {{- if .Values.metadata.persistence.enabled }}
@@ -110,6 +116,7 @@ sumologic/kubelet:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: prometheus
   endpoint: ${SUMO_ENDPOINT_KUBELET_METRICS_SOURCE}
+  compression: {{ .Values.metadata.compression }}
   sending_queue:
     enabled: true
 {{- if .Values.metadata.persistence.enabled }}
@@ -124,6 +131,7 @@ sumologic/node:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: prometheus
   endpoint: ${SUMO_ENDPOINT_NODE_METRICS_SOURCE}
+  compression: {{ .Values.metadata.compression }}
   sending_queue:
     enabled: true
 {{- if .Values.metadata.persistence.enabled }}
@@ -138,6 +146,7 @@ sumologic/scheduler:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: prometheus
   endpoint: ${SUMO_ENDPOINT_SCHEDULER_METRICS_SOURCE}
+  compression: {{ .Values.metadata.compression }}
   sending_queue:
     enabled: true
 {{- if .Values.metadata.persistence.enabled }}
@@ -152,6 +161,7 @@ sumologic/state:
   client: {{ include "sumologic.sumo_client" . }}
   metric_format: prometheus
   endpoint: ${SUMO_ENDPOINT_STATE_METRICS_SOURCE}
+  compression: {{ .Values.metadata.compression }}
   sending_queue:
     enabled: true
 {{- if .Values.metadata.persistence.enabled }}

deploy/helm/sumologic/values.yaml

@@ -1555,6 +1555,9 @@ metadata:
     size: 10Gi
     ## Add custom labels to all otelcol statefulset PVC (logs and metrics)
     pvcLabels: {}
+  
+  ## Compression for logs, metrics and events:
+  compression: zstd
 
   ## Configure metrics pipeline.
   ## This section affects only otelcol provider.