Skip to content

Commit b836ba4

Browse files
adding example to otel traces (#11354) (#11524)
1 parent 51b310b commit b836ba4

File tree

1 file changed

+130
-4
lines changed

1 file changed

+130
-4
lines changed

_data-prepper/pipelines/configuration/sources/otel-trace-source.md

Lines changed: 130 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,22 +43,148 @@ To use the `otel-metrics` source, create the following `pipeline.yaml` file with
4343

4444
```yaml
4545
source:
46-
- otel_trace_source:
46+
otel_trace_source:
4747
```
4848
{% include copy.html %}
4949
5050
If you want to use the OpenTelemetry format for your output, set the `output_format` to `otel`, as shown in the following example:
5151

5252
```yaml
5353
source:
54-
- otel_trace_source:
55-
output_format: otel
54+
otel_trace_source:
55+
output_format: otel
5656
```
5757
{% include copy.html %}
5858

59+
## Example
60+
61+
The following example shows Data Prepper ingesting OTLP traces over HTTPS using a PEM certificate and key with unframed HTTP at a custom path, accepting gzip-compressed payloads, preserving OpenTelemetry document structure, and indexing them into OpenSearch:
62+
63+
```yaml
64+
otel-traces-https:
65+
source:
66+
otel_trace_source:
67+
ssl: true
68+
sslKeyFile: /usr/share/data-prepper/certs/dp-key.pem
69+
sslKeyCertChainFile: /usr/share/data-prepper/certs/dp-cert.pem
70+
unframed_requests: true
71+
path: /ingest/${pipelineName}/v1/traces
72+
compression: gzip
73+
output_format: otel
74+
request_timeout: 15000
75+
health_check_service: true
76+
proto_reflection_service: true
77+
sink:
78+
- opensearch:
79+
hosts: ["https://opensearch:9200"]
80+
index: otel-traces-https
81+
username: admin
82+
password: admin_pass
83+
insecure: true
84+
```
85+
{% include copy.html %}
86+
87+
You can test the pipeline using the following command:
88+
89+
```bash
90+
cat > /tmp/otel-trace3.json <<'JSON'
91+
{
92+
"resourceSpans": [{
93+
"resource": {"attributes":[
94+
{"key":"service.name","value":{"stringValue":"billing"}},
95+
{"key":"service.version","value":{"stringValue":"2.1.0"}}
96+
]},
97+
"scopeSpans": [{
98+
"scope": {"name":"manual-https"},
99+
"spans": [{
100+
"traceId": "1234567890abcdef1234567890abcdef",
101+
"spanId": "feedfacecafebeef",
102+
"name": "PUT /invoice/42",
103+
"startTimeUnixNano": "1739999999000000000",
104+
"endTimeUnixNano": "1740000000000000000",
105+
"attributes": [
106+
{"key":"region","value":{"stringValue":"eu-west-1"}},
107+
{"key":"retry.count","value":{"intValue":"1"}}
108+
]
109+
}]
110+
}]
111+
}]
112+
}
113+
JSON
114+
115+
gzip -c /tmp/otel-trace3.json > /tmp/otel-trace3.json.gz
116+
117+
curl -s -X POST "https://localhost:21890/ingest/otel-traces-https/v1/traces" \
118+
-H 'Content-Type: application/json' \
119+
-H 'Content-Encoding: gzip' \
120+
--insecure \
121+
--data-binary @/tmp/otel-trace3.json.gz
122+
```
123+
{% include copy.html %}
124+
125+
The document stored in OpenSearch contains the following information:
126+
127+
```json
128+
{
129+
...
130+
"hits": {
131+
"total": {
132+
"value": 1,
133+
"relation": "eq"
134+
},
135+
"max_score": 1,
136+
"hits": [
137+
{
138+
"_index": "otel-traces-https",
139+
"_id": "V_5RBpoBqZ1V_u-TvYXf",
140+
"_score": 1,
141+
"_source": {
142+
"traceId": "d76df8e7aefcf7469b71d79fd76df8e7aefcf7469b71d79f",
143+
"droppedLinksCount": 0,
144+
"instrumentationScope": {
145+
"name": "manual-https",
146+
"droppedAttributesCount": 0
147+
},
148+
"resource": {
149+
"schemaUrl": "",
150+
"attributes": {
151+
"service.name": "billing",
152+
"service.version": "2.1.0"
153+
},
154+
"droppedAttributesCount": 0
155+
},
156+
"kind": "SPAN_KIND_UNSPECIFIED",
157+
"droppedEventsCount": 0,
158+
"flags": 0,
159+
"parentSpanId": "",
160+
"schemaUrl": "",
161+
"spanId": "7de79d7da71e71a7de6de79f",
162+
"traceState": "",
163+
"name": "PUT /invoice/42",
164+
"startTime": "2025-02-19T21:19:59Z",
165+
"attributes": {
166+
"retry.count": 1,
167+
"region": "eu-west-1"
168+
},
169+
"links": [],
170+
"endTime": "2025-02-19T21:20:00Z",
171+
"droppedAttributesCount": 0,
172+
"durationInNanos": 1000000000,
173+
"events": [],
174+
"status": {
175+
"code": 0,
176+
"message": ""
177+
}
178+
}
179+
}
180+
]
181+
}
182+
}
183+
```
184+
59185
## Metrics
60186

61-
The 'otel_trace_source' source includes the following metrics.
187+
The `otel_trace_source` source includes the following metrics.
62188

63189
### Counters
64190

0 commit comments

Comments
 (0)