Add more useful metrics to otel collector (#1546)

Author: djeebus

.editorconfig

@@ -0,0 +1,3 @@
+[*.yaml]
+indent_size = 2
+indent_style = space

iac/provider-gcp/.terraform.lock.hcl

@@ -21,56 +21,76 @@ receivers:
   hostmetrics:
     root_path: /hostfs
     collection_interval: 30s
+
+    # all scrapers are added explicitly, so we can be clear
+    # about which metrics we've chosen to include/exclude.
+    # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/hostmetricsreceiver/README.md
     scrapers:
       cpu:
         metrics:
-          system.cpu.time:
-            enabled: false
-          system.cpu.utilization:
+          system.cpu.time:  # counter of seconds spent
+            enabled: true
+          system.cpu.frequency:
             enabled: false
           system.cpu.logical.count:
             enabled: true
           system.cpu.physical.count:
             enabled: true
+          system.cpu.utilization:  # percentage utilized in the moment
+            enabled: false
+
+      # entirely disabled, so we omit the module
+      #disk:
+      #  metrics:
+      #    system.disk.io:
+      #      enabled: false
+      #    system.disk.io_time:
+      #      enabled: false
+      #    system.disk.merged:
+      #      enabled: false
+      #    system.disk.operation_time:
+      #      enabled: false
+      #    system.disk.operations:
+      #      enabled: false
+      #    system.disk.pending_operations:
+      #      enabled: false
+      #    system.disk.weighted_io_time:
+      #      enabled: false
 
       load:
         metrics:
+          system.cpu.load_average.15m:
+            enabled: true
           system.cpu.load_average.1m:
             enabled: true
           system.cpu.load_average.5m:
             enabled: true
-          system.cpu.load_average.15m:
-            enabled: false
 
-      # Scraping traffic going in/out of network interfaces
-      network:
+      #  filter not needed stuff (mode, type)
+      filesystem:
         metrics:
-          system.network.connections:
-            enabled: false
-          system.network.dropped:
-            enabled: false
-          system.network.errors:
-            enabled: false
-          system.network.io:
-              enabled: true
-          system.network.packets:
-            enabled: false
+          system.filesystem.inodes.usage:
+            enabled: true
+          system.filesystem.usage:
+            enabled: true
+          system.filesystem.utilization:
+            enabled: true
 
       # Scraping only total memory (memory limit) and available memory to reduce cardinality
       memory:
         metrics:
-          system.linux.memory.dirty:
-            enabled: false
+          system.memory.usage:
+            enabled: true
           system.linux.memory.available: # free memory
             enabled: true
+          system.linux.memory.dirty:  # data waiting to be written to disk
+            enabled: true
           system.memory.limit: # total memory
             enabled: true
           system.memory.page_size:
             enabled: false
-          system.memory.usage:
-            enabled: false
           system.memory.utilization:
-            enabled: false
+            enabled: true
 
           # Huge pages metrics added as part of temporal patch
           # https://github.com/e2b-dev/opentelemetry-collector-contrib/pull/1
@@ -85,14 +105,80 @@ receivers:
           system.linux.memory.huge_pages.total:
             enabled: true
 
-      #  filter not needed stuff (mode, type)
-      #  filter /dev/loop*, maybe check fo other that are polluting
-      filesystem:
+      # Scraping traffic going in/out of network interfaces
+      network:
         metrics:
-          system.filesystem.inodes.usage:
+          system.network.connections:
+            enabled: true
+          system.network.dropped:
+            enabled: true
+          system.network.errors:
+            enabled: true
+          system.network.io:
+            enabled: true
+          system.network.packets:
             enabled: false
-          system.filesystem.usage:
+          system.network.conntrack.count:
+            enabled: true
+          system.network.conntrack.max:
+            enabled: true
+
+      # entirely disabled, so we omit the module
+      #paging:
+      #  metrics:
+      #    system.paging.faults:
+      #      enabled: false
+      #    system.paging.operations:
+      #      enabled: false
+      #    system.paging.usage:
+      #      enabled: false
+      #    system.paging.utilization:
+      #      enabled: false
+
+      processes:
+        metrics:
+          system.processes.count:
             enabled: true
+          system.processes.created:
+            enabled: true
+
+      # entirely disabled, so we omit the module
+      #process:
+      #  metrics:
+      #    process.cpu.time:
+      #      enabled: false
+      #    process.disk.io:
+      #      enabled: false
+      #    process.memory.usage:
+      #      enabled: false
+      #    process.memory.virtual:
+      #      enabled: false
+      #    process.context_switches:
+      #      enabled: false
+      #    process.cpu.utilization:
+      #      enabled: false
+      #    process.disk.operations:
+      #      enabled: false
+      #    process.handles:
+      #      enabled: false
+      #    process.memory.utilization:
+      #      enabled: false
+      #    process.open_file_descriptors:
+      #      enabled: false
+      #    process.paging.faults:
+      #      enabled: false
+      #    process.signals_pending:
+      #      enabled: false
+      #    process.threads:
+      #      enabled: false
+      #    process.uptime:
+      #      enabled: false
+
+      # entirely disabled, so we omit the module
+      #system:
+      #  metrics:
+      #    system.uptime:
+      #      enabled: false
 
 processors:
   batch:
@@ -107,14 +193,15 @@ processors:
     metrics:
       datapoint:
         # Drop system.network.io for veth-* or docker* interfaces
-        - 'metric.name == "system.network.io" and IsMatch(attributes["device"], "^(veth-.*|docker.*|lo)$")'
-        # Drop system.filesystem.usage for loop devices
-        - 'metric.name == "system.filesystem.usage" and IsMatch(attributes["device"], "^/dev/loop.*$")'
+        - 'IsMatch(metric.name, "system.network..*") and IsMatch(attributes["device"], "^(veth.*|docker.*|lo)$")'
+        # Drop anything related to loop devices
+        - 'IsMatch(attributes["device"], "^/dev/loop.*$")'
 
   attributes/strip_fs_labels:
     include:
       match_type: strict
-      metric_names: [system.filesystem.usage]
+      metric_names:
+        - system.filesystem.usage
     actions:
       - action: delete
         key: mode
@@ -143,7 +230,6 @@ processors:
           - "vault.*"
           - "client_proxy.*"
           - "Click*"
-          - "otelcol.*"
           - "pgxpool.*"
 
   filter/prometheus:
@@ -180,6 +266,18 @@ processors:
             aggregation_type: sum
             label_set: [instance, node_id, node_status, node_pool]
 
+  metricstransform/single_cpu:
+    transforms:
+      - include: "system.cpu.time"
+        match_type: strict
+        action: update
+        operations:
+          - action: aggregate_labels
+            label_set:
+              - node.id
+              - state
+            aggregation_type: sum
+
   resourcedetection:
     # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor
     detectors: [gcp]
@@ -280,7 +378,12 @@ service:
     metrics/prometheus:
       receivers:
         - prometheus
-      processors: [filter/prometheus, metricstransform, resourcedetection, transform/set-name, batch]
+      processors:
+        - filter/prometheus
+        - metricstransform
+        - resourcedetection
+        - transform/set-name
+        - batch
       exporters:
         - otlphttp/grafana_cloud
     metrics/rpc_only:
@@ -292,7 +395,12 @@ service:
     metrics/host:
       receivers:
         - hostmetrics
-      processors: [filter/drop_by_device, attributes/strip_fs_labels, attributes/host_metrics_node, batch]
+      processors:
+        - filter/drop_by_device
+        - attributes/strip_fs_labels
+        - attributes/host_metrics_node
+        - metricstransform/single_cpu
+        - batch
       exporters:
         - otlphttp/grafana_cloud
     metrics/external:

iac/provider-gcp/nomad-cluster-disk-image/.terraform.lock.hcl

@@ -62,7 +62,6 @@ job "otel-collector" {
         ]
         args = [
           "--config=local/config/otel-collector-config.yaml",
-          "--feature-gates=pkg.translator.prometheus.NormalizeName",
         ]
 
         ports = [
@@ -91,4 +90,4 @@ EOF
       }
     }
   }
-}
+}