chore(dev): add opentelemetry traces e2e tests

.github/workflows/changes.yml

@@ -143,6 +143,8 @@ on:
         value: ${{ jobs.e2e_tests.outputs.opentelemetry-logs }}
       e2e-opentelemetry-metrics:
         value: ${{ jobs.e2e_tests.outputs.opentelemetry-metrics }}
+      e2e-opentelemetry-traces:
+        value: ${{ jobs.e2e_tests.outputs.opentelemetry-traces }}
       int-tests-any:
         value: ${{ jobs.int_tests.outputs.any }}
       e2e-tests-any:
@@ -403,6 +405,7 @@ jobs:
       datadog-metrics: ${{ steps.filter.outputs.datadog-metrics }}
       opentelemetry-logs: ${{ steps.filter.outputs.opentelemetry-logs }}
       opentelemetry-metrics: ${{ steps.filter.outputs.opentelemetry-metrics }}
+      opentelemetry-traces: ${{ steps.filter.outputs.opentelemetry-traces }}
       any: ${{ steps.detect-changes.outputs.any }}
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
@@ -434,7 +437,8 @@ jobs:
             "datadog-logs": ${{ steps.filter.outputs.datadog-logs }},
             "datadog-metrics": ${{ steps.filter.outputs.datadog-metrics }},
             "opentelemetry-logs": ${{ steps.filter.outputs.opentelemetry-logs }},
-            "opentelemetry-metrics": ${{ steps.filter.outputs.opentelemetry-metrics }}
+            "opentelemetry-metrics": ${{ steps.filter.outputs.opentelemetry-metrics }},
+            "opentelemetry-traces": ${{ steps.filter.outputs.opentelemetry-traces }}
           }
           EOF
           )

.github/workflows/ci-integration-review.yml

@@ -136,7 +136,7 @@ jobs:
     strategy:
       matrix:
         service: [
-          "datadog-logs", "datadog-metrics", "opentelemetry-logs", "opentelemetry-metrics"
+          "datadog-logs", "datadog-metrics", "opentelemetry-logs", "opentelemetry-metrics", "opentelemetry-traces"
         ]
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
@@ -162,7 +162,6 @@ jobs:
           max_attempts: 3
           command: bash scripts/run-integration-test.sh e2e ${{ matrix.service }}
 
-
   update-pr-status:
     name: Signal result to PR
     runs-on: ubuntu-24.04

.github/workflows/integration.yml

@@ -136,7 +136,7 @@ jobs:
     if: ${{ !failure() && !cancelled() && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch') }}
     strategy:
       matrix:
-        service: [ "datadog-logs", "datadog-metrics", "opentelemetry-logs", "opentelemetry-metrics" ]
+        service: [ "datadog-logs", "datadog-metrics", "opentelemetry-logs", "opentelemetry-metrics", "opentelemetry-traces" ]
     timeout-minutes: 90
 
     steps:

scripts/e2e/opentelemetry-traces/compose.yaml

@@ -0,0 +1,92 @@
+name: opentelemetry-vector-e2e
+services:
+  otel-collector-source:
+    container_name: otel-collector-source
+    image: otel/opentelemetry-collector-contrib:${CONFIG_COLLECTOR_VERSION}
+    init: true
+    user: "0:0" # test only, override special user with root
+    volumes:
+      - type: bind
+        source: ../../../tests/data/e2e/opentelemetry/traces/collector-source.yaml
+        target: /etc/otelcol-contrib/config.yaml
+        read_only: true
+      - type: volume
+        source: vector_target
+        target: /output
+    ports:
+      - "${OTEL_COLLECTOR_SOURCE_GRPC_PORT:-4317}:4317"
+      - "${OTEL_COLLECTOR_SOURCE_HTTP_PORT:-4318}:4318"
+    command: [ "--config=/etc/otelcol-contrib/config.yaml" ]
+
+  traces-generator:
+    container_name: traces-generator
+    build:
+      context: ../../../
+      dockerfile: ./scripts/e2e/opentelemetry-common/telemetrygen.Dockerfile
+    init: true
+    depends_on:
+      otel-collector-source:
+        condition: service_started
+      vector:
+        condition: service_started
+      otel-collector-sink:
+        condition: service_started
+    command:
+      - "-c"
+      - |
+        until nc -z otel-collector-source 4318; do
+          sleep 0.5
+        done
+        telemetrygen traces \
+          --otlp-endpoint=otel-collector-source:4318 \
+          --otlp-insecure \
+          --otlp-http \
+          --traces=100 \
+          --rate=0
+
+  otel-collector-sink:
+    container_name: otel-collector-sink
+    build:
+      context: ../../../
+      dockerfile: ./scripts/e2e/opentelemetry-common/collector.Dockerfile
+      args:
+        CONFIG_COLLECTOR_VERSION: ${CONFIG_COLLECTOR_VERSION}
+    init: true
+    user: "0:0" # test only, override special user with root
+    command: [ "--config", "/etc/otelcol-contrib/config.yaml" ]
+    volumes:
+      - type: bind
+        source: ../../../tests/data/e2e/opentelemetry/traces/collector-sink.yaml
+        target: /etc/otelcol-contrib/config.yaml
+        read_only: true
+      - type: volume
+        source: vector_target
+        target: /output
+    ports:
+      - "${OTEL_COLLECTOR_SINK_HTTP_PORT:-5318}:5318"
+
+  vector:
+    container_name: vector-otel-traces-e2e
+    image: ${CONFIG_VECTOR_IMAGE}
+    init: true
+    volumes:
+      - type: bind
+        source: ../../../tests/data/e2e/opentelemetry/traces/vector_otlp.yaml
+        target: /etc/vector/vector.yaml
+        read_only: true
+      - type: volume
+        source: vector_target
+        target: /output
+    environment:
+      - VECTOR_LOG=${VECTOR_LOG:-info}
+      - FEATURES=e2e-tests-opentelemetry
+    command: [ "vector", "-c", "/etc/vector/vector.yaml" ]
+
+volumes:
+  vector_target:
+    external: true
+
+networks:
+  default:
+    name: ${VECTOR_NETWORK}
+    external: true

scripts/e2e/opentelemetry-traces/test.yaml

@@ -0,0 +1,26 @@
+features:
+  - e2e-tests-opentelemetry
+
+test: "e2e"
+
+test_filter: "opentelemetry::traces::"
+
+runner:
+  needs_docker_socket: true
+  env:
+    OTEL_COLLECTOR_SOURCE_GRPC_PORT: '4317'
+    OTEL_COLLECTOR_SOURCE_HTTP_PORT: '4318'
+    OTEL_COLLECTOR_SINK_HTTP_PORT: '5318'
+
+matrix:
+  # Determines which `otel/opentelemetry-collector-contrib` version to use
+  collector_version: [ 'latest' ]
+
+# Only trigger this integration test if relevant OTEL source/sink files change
+paths:
+  - "src/sources/opentelemetry/**"
+  - "src/sinks/opentelemetry/**"
+  - "src/internal_events/opentelemetry_*"
+  - "tests/e2e/opentelemetry/traces/**"
+  - "scripts/e2e/opentelemetry-traces/**"
+  - "lib/codecs/src/**/otlp.rs"

tests/data/e2e/opentelemetry/traces/collector-sink.yaml

@@ -0,0 +1,26 @@
+receivers:
+  otlp:
+    protocols:
+      http:
+        endpoint: "0.0.0.0:5318"
+
+processors:
+  batch: { }
+
+exporters:
+  debug: { }
+  file:
+    path: /output/opentelemetry-traces/collector-file-exporter.log
+    rotation:
+      max_megabytes: 10
+      max_days: 1
+
+service:
+  pipelines:
+    traces:
+      receivers: [ otlp ]
+      processors: [ batch ]
+      exporters: [ debug, file ]
+  telemetry:
+    logs:
+      level: "debug"

tests/data/e2e/opentelemetry/traces/collector-source.yaml

@@ -0,0 +1,32 @@
+receivers:
+  otlp:
+    protocols:
+      http:
+        endpoint: "0.0.0.0:4318"
+      grpc:
+
+processors:
+  batch: { }
+
+exporters:
+  otlp/grpc:
+    endpoint: vector:4317
+    tls:
+      insecure: true
+  otlphttp/vector:
+    endpoint: http://vector:4318
+    tls:
+      insecure: true
+  debug: { }
+  file:
+    path: /output/opentelemetry-traces/collector-source-file-exporter.log
+    rotation:
+      max_megabytes: 10
+      max_days: 1
+
+service:
+  pipelines:
+    traces:
+      receivers: [ otlp ]
+      processors: [ batch ]
+      exporters: [ debug, otlp/grpc, otlphttp/vector, file ]

tests/data/e2e/opentelemetry/traces/vector_otlp.yaml

@@ -0,0 +1,42 @@
+sources:
+  source0:
+    type: opentelemetry
+    grpc:
+      address: 0.0.0.0:4317
+    http:
+      address: 0.0.0.0:4318
+      keepalive:
+        max_connection_age_jitter_factor: 0.1
+        max_connection_age_secs: 300
+    use_otlp_decoding: true
+
+  internal_metrics:
+    type: internal_metrics
+    scrape_interval_secs: 60
+
+sinks:
+  otel_sink:
+    inputs:
+      - source0.traces
+    type: opentelemetry
+    protocol:
+      type: http
+      uri: http://otel-collector-sink:5318/v1/traces
+      encoding:
+        codec: otlp
+
+  otel_file_sink:
+    type: file
+    path: "/output/opentelemetry-traces/vector-file-sink.log"
+    inputs:
+      - source0.traces
+    encoding:
+      codec: json
+
+  metrics_file_sink:
+    type: file
+    path: "/output/opentelemetry-traces/vector-internal-metrics-sink.log"
+    inputs:
+      - internal_metrics
+    encoding:
+      codec: json

tests/e2e/opentelemetry/mod.rs

@@ -1,5 +1,6 @@
 pub mod logs;
 pub mod metrics;
+pub mod traces;
 
 use std::{io, path::Path, process::Command};