From cc520b70559d4a55f91687f94a426531f3d8254d Mon Sep 17 00:00:00 2001 From: Anton Rubin Date: Tue, 21 Oct 2025 11:33:03 +0100 Subject: [PATCH 1/4] adding example to otel traces Signed-off-by: Anton Rubin --- .../sources/otel-trace-source.md | 134 +++++++++++++++++- 1 file changed, 130 insertions(+), 4 deletions(-) diff --git a/_data-prepper/pipelines/configuration/sources/otel-trace-source.md b/_data-prepper/pipelines/configuration/sources/otel-trace-source.md index 517020cfbb8..83533566be3 100644 --- a/_data-prepper/pipelines/configuration/sources/otel-trace-source.md +++ b/_data-prepper/pipelines/configuration/sources/otel-trace-source.md @@ -43,7 +43,7 @@ To use the `otel-metrics` source, create the following `pipeline.yaml` file with ```yaml source: - - otel_trace_source: + otel_trace_source: ``` {% include copy.html %} @@ -51,14 +51,140 @@ If you want to use the OpenTelemetry format for your output, set the `output_for ```yaml source: - - otel_trace_source: - output_format: otel + otel_trace_source: + output_format: otel ``` {% include copy.html %} +## Example + +The following example demonstrates Data Prepper ingesting OTLP traces over HTTPS using a PEM cert and key with unframed HTTP at a custom path, accepting gzip payloads, preserving OTel shaped documents, and indexing them into OpenSearch: + +```yaml +otel-traces-https: + source: + otel_trace_source: + ssl: true + sslKeyFile: "/usr/share/data-prepper/certs/dp-key.pem" + sslKeyCertChainFile: "/usr/share/data-prepper/certs/dp-cert.pem" + unframed_requests: true + path: "/ingest/${pipelineName}/v1/traces" + compression: gzip + output_format: otel + request_timeout: 15000 + health_check_service: true + proto_reflection_service: true + sink: + - opensearch: + hosts: ["https://opensearch:9200"] + index: "otel-traces-https" + username: "admin" + password: "admin_pass" + insecure: true +``` +{% include copy.html %} + +You can test the pipeline using the following command: + +```bash +cat > /tmp/otel-trace3.json <<'JSON' +{ + "resourceSpans": [{ + "resource": {"attributes":[ + {"key":"service.name","value":{"stringValue":"billing"}}, + {"key":"service.version","value":{"stringValue":"2.1.0"}} + ]}, + "scopeSpans": [{ + "scope": {"name":"manual-https"}, + "spans": [{ + "traceId": "1234567890abcdef1234567890abcdef", + "spanId": "feedfacecafebeef", + "name": "PUT /invoice/42", + "startTimeUnixNano": "1739999999000000000", + "endTimeUnixNano": "1740000000000000000", + "attributes": [ + {"key":"region","value":{"stringValue":"eu-west-1"}}, + {"key":"retry.count","value":{"intValue":"1"}} + ] + }] + }] + }] +} +JSON + +gzip -c /tmp/otel-trace3.json > /tmp/otel-trace3.json.gz + +curl -s -X POST "https://localhost:21890/ingest/otel-traces-https/v1/traces" \ + -H 'Content-Type: application/json' \ + -H 'Content-Encoding: gzip' \ + --insecure \ + --data-binary @/tmp/otel-trace3.json.gz +``` +{% include copy.html %} + +The document stored in OpenSearch contains the following information: + +```json +{ + ... + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 1, + "hits": [ + { + "_index": "otel-traces-https", + "_id": "V_5RBpoBqZ1V_u-TvYXf", + "_score": 1, + "_source": { + "traceId": "d76df8e7aefcf7469b71d79fd76df8e7aefcf7469b71d79f", + "droppedLinksCount": 0, + "instrumentationScope": { + "name": "manual-https", + "droppedAttributesCount": 0 + }, + "resource": { + "schemaUrl": "", + "attributes": { + "service.name": "billing", + "service.version": "2.1.0" + }, + "droppedAttributesCount": 0 + }, + "kind": "SPAN_KIND_UNSPECIFIED", + "droppedEventsCount": 0, + "flags": 0, + "parentSpanId": "", + "schemaUrl": "", + "spanId": "7de79d7da71e71a7de6de79f", + "traceState": "", + "name": "PUT /invoice/42", + "startTime": "2025-02-19T21:19:59Z", + "attributes": { + "retry.count": 1, + "region": "eu-west-1" + }, + "links": [], + "endTime": "2025-02-19T21:20:00Z", + "droppedAttributesCount": 0, + "durationInNanos": 1000000000, + "events": [], + "status": { + "code": 0, + "message": "" + } + } + } + ] + } +} +``` + ## Metrics -The 'otel_trace_source' source includes the following metrics. +The `otel_trace_source` source includes the following metrics. ### Counters From 21722cfa347640c702d70f7797622677e87d8d9b Mon Sep 17 00:00:00 2001 From: AntonEliatra Date: Tue, 4 Nov 2025 10:42:12 +0000 Subject: [PATCH 2/4] Update otel-trace-source.md Signed-off-by: AntonEliatra --- .../pipelines/configuration/sources/otel-trace-source.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/_data-prepper/pipelines/configuration/sources/otel-trace-source.md b/_data-prepper/pipelines/configuration/sources/otel-trace-source.md index 83533566be3..ba8fed485b7 100644 --- a/_data-prepper/pipelines/configuration/sources/otel-trace-source.md +++ b/_data-prepper/pipelines/configuration/sources/otel-trace-source.md @@ -65,10 +65,10 @@ otel-traces-https: source: otel_trace_source: ssl: true - sslKeyFile: "/usr/share/data-prepper/certs/dp-key.pem" - sslKeyCertChainFile: "/usr/share/data-prepper/certs/dp-cert.pem" + sslKeyFile: /usr/share/data-prepper/certs/dp-key.pem + sslKeyCertChainFile: /usr/share/data-prepper/certs/dp-cert.pem unframed_requests: true - path: "/ingest/${pipelineName}/v1/traces" + path: /ingest/${pipelineName}/v1/traces compression: gzip output_format: otel request_timeout: 15000 @@ -77,7 +77,7 @@ otel-traces-https: sink: - opensearch: hosts: ["https://opensearch:9200"] - index: "otel-traces-https" + index: otel-traces-https username: "admin" password: "admin_pass" insecure: true From 1574ad8f9ff4f66cc51e43bd34ae412271a85b00 Mon Sep 17 00:00:00 2001 From: AntonEliatra Date: Tue, 4 Nov 2025 11:47:31 +0000 Subject: [PATCH 3/4] Update otel-trace-source.md Signed-off-by: AntonEliatra --- .../pipelines/configuration/sources/otel-trace-source.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_data-prepper/pipelines/configuration/sources/otel-trace-source.md b/_data-prepper/pipelines/configuration/sources/otel-trace-source.md index ba8fed485b7..6637d7cb937 100644 --- a/_data-prepper/pipelines/configuration/sources/otel-trace-source.md +++ b/_data-prepper/pipelines/configuration/sources/otel-trace-source.md @@ -78,8 +78,8 @@ otel-traces-https: - opensearch: hosts: ["https://opensearch:9200"] index: otel-traces-https - username: "admin" - password: "admin_pass" + username: admin + password: admin_pass insecure: true ``` {% include copy.html %} From 9c850d048c6fe8ee5fc9dea8f7846f13b3a82121 Mon Sep 17 00:00:00 2001 From: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Date: Tue, 11 Nov 2025 11:03:14 -0500 Subject: [PATCH 4/4] Apply suggestions from code review Signed-off-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> --- .../pipelines/configuration/sources/otel-trace-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data-prepper/pipelines/configuration/sources/otel-trace-source.md b/_data-prepper/pipelines/configuration/sources/otel-trace-source.md index 6637d7cb937..13fb64b7357 100644 --- a/_data-prepper/pipelines/configuration/sources/otel-trace-source.md +++ b/_data-prepper/pipelines/configuration/sources/otel-trace-source.md @@ -58,7 +58,7 @@ source: ## Example -The following example demonstrates Data Prepper ingesting OTLP traces over HTTPS using a PEM cert and key with unframed HTTP at a custom path, accepting gzip payloads, preserving OTel shaped documents, and indexing them into OpenSearch: +The following example shows Data Prepper ingesting OTLP traces over HTTPS using a PEM certificate and key with unframed HTTP at a custom path, accepting gzip-compressed payloads, preserving OpenTelemetry document structure, and indexing them into OpenSearch: ```yaml otel-traces-https: