diff --git a/cmd/apmsoak/scenarios.yml b/cmd/apmsoak/scenarios.yml index c9cfb37..cb0e638 100644 --- a/cmd/apmsoak/scenarios.yml +++ b/cmd/apmsoak/scenarios.yml @@ -28,3 +28,6 @@ scenarios: - event_rate: 10000/1s agent_name: apm-ruby agents_replicas: 3 + apm-server-otlp: + - event_rate: 100/s + agent_name: otlp-traces diff --git a/internal/loadgen/eventhandler.go b/internal/loadgen/eventhandler.go index d5f41b0..359be3b 100644 --- a/internal/loadgen/eventhandler.go +++ b/internal/loadgen/eventhandler.go @@ -22,7 +22,7 @@ import ( // events holds the current stored events. // -//go:embed events/*.ndjson +//go:embed events/*.ndjson events/*.jsonl var events embed.FS type EventHandlerParams struct { diff --git a/internal/loadgen/eventhandler/otlp-collector.go b/internal/loadgen/eventhandler/otlp-collector.go new file mode 100644 index 0000000..9bf4dc8 --- /dev/null +++ b/internal/loadgen/eventhandler/otlp-collector.go @@ -0,0 +1,88 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package eventhandler + +import ( + "bytes" + "fmt" + "strconv" + "time" + + "github.com/tidwall/gjson" +) + +type OTLPEventCollector struct{} + +// Filter skips processing RUM related events. +func (a *OTLPEventCollector) Filter(line []byte) error { + if bytes.HasPrefix(line, rumMetaHeader) { + return fmt.Errorf("rum data support not implemented") + } + + return nil +} + +// IsMeta identifies metadata lines from APM protocol. +func (a *OTLPEventCollector) IsMeta(line []byte) bool { + return bytes.HasPrefix(line, metaHeader) +} + +// Process processes single lines extracting APM events. +// It uniforms events timestamp. +func (a *OTLPEventCollector) Process(linecopy []byte) event { + event := event{payload: linecopy} + result := gjson.ParseBytes(linecopy) + + result.ForEach(func(key, value gjson.Result) bool { + event.objectType = key.Str // lines look like {"span":{...}} + + switch event.objectType { + case "resourceLogs": + // compute minimum timestamp from all resource logs + value.Get("#.scopeLogs.#.logRecords"). + ForEach(func(key, value gjson.Result) bool { + value.ForEach(func(key, value gjson.Result) bool { + value.ForEach(func(key, value gjson.Result) bool { + s, err := strconv.ParseInt(value.Get("timeUnixNano").String(), 10, 64) + if err != nil { + return true + } + t := time.Unix(0, s) + if event.timestamp.IsZero() || t.Before(event.timestamp) { + event.timestamp = t + } + return true + }) + return true + }) + return true + }) + } + + // timestampResult := value.Get("timestamp") + // if timestampResult.Exists() { + // switch timestampResult.Type { + // case gjson.Number: + // us := timestampResult.Int() + // if us >= 0 { + // s := us / 1000000 + // ns := (us - (s * 1000000)) * 1000 + // event.timestamp = time.Unix(s, ns) + // } + // case gjson.String: + // tstr := timestampResult.Str + // for _, f := range supportedTSFormats { + // if t, err := time.Parse(f, tstr); err == nil { + // event.timestamp = t + // break + // } + // } + // } + // } + return true + }) + + return event +} diff --git a/internal/loadgen/eventhandler/otlp-collector_test.go b/internal/loadgen/eventhandler/otlp-collector_test.go new file mode 100644 index 0000000..9d4e756 --- /dev/null +++ b/internal/loadgen/eventhandler/otlp-collector_test.go @@ -0,0 +1,29 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package eventhandler + +import ( + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestOTLPEventCollector_Process(t *testing.T) { + o := OTLPEventCollector{} + + minTimestamp := "1581452773000000789" + s, err := strconv.ParseInt(minTimestamp, 10, 64) + assert.NoError(t, err) + et := time.Unix(0, s) + + line := `{"resourceLogs":[{"resource":{"attributes":[{"key":"resource-attr","value":{"stringValue":"resource-attr-val-1"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"timeUnixNano":"1581452773000009875","severityNumber":9,"severityText":"Info","name":"logA","body":{"stringValue":"This is a log message"},"attributes":[{"key":"app","value":{"stringValue":"server"}},{"key":"instance_num","value":{"intValue":"1"}}],"droppedAttributesCount":1,"traceId":"08040201000000000000000000000000","spanId":"0102040800000000"},{"timeUnixNano":"1581452773000000789","severityNumber":9,"severityText":"Info","name":"logB","body":{"stringValue":"something happened"},"attributes":[{"key":"customer","value":{"stringValue":"acme"}},{"key":"env","value":{"stringValue":"dev"}}],"droppedAttributesCount":1,"traceId":"","spanId":""}]}]}]}` + + event := o.Process([]byte(line)) + + assert.Equal(t, "resourceLogs", event.objectType) + assert.Equal(t, et, event.timestamp) +} diff --git a/internal/loadgen/eventhandler/otlp.go b/internal/loadgen/eventhandler/otlp.go new file mode 100644 index 0000000..532d105 --- /dev/null +++ b/internal/loadgen/eventhandler/otlp.go @@ -0,0 +1,72 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package eventhandler + +import ( + "context" + "fmt" + "io" + "net/http" +) + +type OTLPTransport struct { + client *http.Client + headers http.Header + url string +} + +func (o *OTLPTransport) SendEvents(ctx context.Context, r io.Reader, ignoreErrs bool) error { + req, err := http.NewRequestWithContext(ctx, "POST", o.url, r) + if err != nil { + return err + } + req.Header = o.headers + + res, err := o.client.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + + if !ignoreErrs { + switch res.StatusCode / 100 { + case 4: + return fmt.Errorf("unexpected client error: %d", res.StatusCode) + case 5: + return fmt.Errorf("unexpected server error: %d", res.StatusCode) + } + } + + return nil +} + +func newOTLPTransport(c *http.Client, srvURL, u, apiKey string, headers map[string]string) *OTLPTransport { + h := make(http.Header) + h.Set("Content-Encoding", "deflate") + h.Set("Content-Type", "application/x-json") + h.Set("Authorization", "ApiKey "+apiKey) + + for name, header := range headers { + h.Set(name, header) + } + + return &OTLPTransport{ + client: c, + url: srvURL + u, + headers: h, + } +} + +func NewOTLPLogsTransport() { + panic("not implemented") +} + +func NewOTLPMetricsTransport(c *http.Client, srvURL, apiKey string, headers map[string]string) Transport { + return newOTLPTransport(c, srvURL, "/v1/metrics", apiKey, headers) +} + +func NewOTLPTracesTransport(c *http.Client, srvURL, apiKey string, headers map[string]string) Transport { + return newOTLPTransport(c, srvURL, "/v1/traces", apiKey, headers) +} diff --git a/internal/loadgen/events/otlp-logs-0.0.0.jsonl b/internal/loadgen/events/otlp-logs-0.0.0.jsonl new file mode 100644 index 0000000..7f98bdf --- /dev/null +++ b/internal/loadgen/events/otlp-logs-0.0.0.jsonl @@ -0,0 +1,5 @@ +{"resourceLogs":[{"resource":{"attributes":[{"key":"resource-attr","value":{"stringValue":"resource-attr-val-1"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"timeUnixNano":"1581452773000000789","severityNumber":9,"severityText":"Info","name":"logA","body":{"stringValue":"This is a log message"},"attributes":[{"key":"app","value":{"stringValue":"server"}},{"key":"instance_num","value":{"intValue":"1"}}],"droppedAttributesCount":1,"traceId":"08040201000000000000000000000000","spanId":"0102040800000000"},{"timeUnixNano":"1581452773000000789","severityNumber":9,"severityText":"Info","name":"logB","body":{"stringValue":"something happened"},"attributes":[{"key":"customer","value":{"stringValue":"acme"}},{"key":"env","value":{"stringValue":"dev"}}],"droppedAttributesCount":1,"traceId":"","spanId":""}]}]}]} +{"resourceLogs":[{"resource":{"attributes":[{"key":"resource-attr","value":{"stringValue":"resource-attr-val-1"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"timeUnixNano":"1581452773000001233","severityNumber":9,"severityText":"Info","name":"logA","body":{"stringValue":"This is a log message"},"attributes":[{"key":"app","value":{"stringValue":"server"}},{"key":"instance_num","value":{"intValue":"1"}}],"droppedAttributesCount":1,"traceId":"08040201000000000000000000000000","spanId":"0102040800000000"},{"timeUnixNano":"1581452773000000789","severityNumber":9,"severityText":"Info","name":"logB","body":{"stringValue":"something happened"},"attributes":[{"key":"customer","value":{"stringValue":"acme"}},{"key":"env","value":{"stringValue":"dev"}}],"droppedAttributesCount":1,"traceId":"","spanId":""}]}]}]} +{"resourceLogs":[{"resource":{"attributes":[{"key":"resource-attr","value":{"stringValue":"resource-attr-val-1"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"timeUnixNano":"1581452773000005443","severityNumber":9,"severityText":"Info","name":"logA","body":{"stringValue":"This is a log message"},"attributes":[{"key":"app","value":{"stringValue":"server"}},{"key":"instance_num","value":{"intValue":"1"}}],"droppedAttributesCount":1,"traceId":"08040201000000000000000000000000","spanId":"0102040800000000"},{"timeUnixNano":"1581452773000000789","severityNumber":9,"severityText":"Info","name":"logB","body":{"stringValue":"something happened"},"attributes":[{"key":"customer","value":{"stringValue":"acme"}},{"key":"env","value":{"stringValue":"dev"}}],"droppedAttributesCount":1,"traceId":"","spanId":""}]}]}]} +{"resourceLogs":[{"resource":{"attributes":[{"key":"resource-attr","value":{"stringValue":"resource-attr-val-1"}}]},"scopeLogs":[{"scope":{},"logRecords":[{"timeUnixNano":"1581452773000009875","severityNumber":9,"severityText":"Info","name":"logA","body":{"stringValue":"This is a log message"},"attributes":[{"key":"app","value":{"stringValue":"server"}},{"key":"instance_num","value":{"intValue":"1"}}],"droppedAttributesCount":1,"traceId":"08040201000000000000000000000000","spanId":"0102040800000000"},{"timeUnixNano":"1581452773000000789","severityNumber":9,"severityText":"Info","name":"logB","body":{"stringValue":"something happened"},"attributes":[{"key":"customer","value":{"stringValue":"acme"}},{"key":"env","value":{"stringValue":"dev"}}],"droppedAttributesCount":1,"traceId":"","spanId":""}]}]}]} + diff --git a/internal/loadgen/events/otlp-metrics-0.0.0.jsonl b/internal/loadgen/events/otlp-metrics-0.0.0.jsonl new file mode 100644 index 0000000..6ba3e63 --- /dev/null +++ b/internal/loadgen/events/otlp-metrics-0.0.0.jsonl @@ -0,0 +1,5 @@ +{"resourceMetrics":[{"resource":{"attributes":[{"key":"resource-attr","value":{"stringValue":"resource-attr-val-1"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"counter-int","unit":"1","sum":{"dataPoints":[{"attributes":[{"key":"label-1","value":{"stringValue":"label-value-1"}}],"startTimeUnixNano":"1581452773000000789","timeUnixNano":"1581452773000000789","asInt":"123"},{"attributes":[{"key":"label-2","value":{"stringValue":"label-value-2"}}],"startTimeUnixNano":"1581452772000000321","timeUnixNano":"1581452773000000789","asInt":"456"}],"aggregationTemporality":2,"isMonotonic":true}},{"name":"counter-int","unit":"1","sum":{"dataPoints":[{"attributes":[{"key":"label-1","value":{"stringValue":"label-value-1"}}],"startTimeUnixNano":"1581452772000000321","timeUnixNano":"1581452773000000789","asInt":"123"},{"attributes":[{"key":"label-2","value":{"stringValue":"label-value-2"}}],"startTimeUnixNano":"1581452772000000321","timeUnixNano":"1581452773000000789","asInt":"456"}],"aggregationTemporality":2,"isMonotonic":true}}]}]}]} +{"resourceMetrics":[{"resource":{"attributes":[{"key":"resource-attr","value":{"stringValue":"resource-attr-val-1"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"counter-int","unit":"1","sum":{"dataPoints":[{"attributes":[{"key":"label-1","value":{"stringValue":"label-value-1"}}],"startTimeUnixNano":"1581452773000001459","timeUnixNano":"1581452773000001459","asInt":"120"},{"attributes":[{"key":"label-2","value":{"stringValue":"label-value-2"}}],"startTimeUnixNano":"1581452773000001459","timeUnixNano":"1581452773000001459","asInt":"456"}],"aggregationTemporality":2,"isMonotonic":true}},{"name":"counter-int","unit":"1","sum":{"dataPoints":[{"attributes":[{"key":"label-1","value":{"stringValue":"label-value-1"}}],"startTimeUnixNano":"1581452773000001459","timeUnixNano":"1581452773000001459","asInt":"123"},{"attributes":[{"key":"label-2","value":{"stringValue":"label-value-2"}}],"startTimeUnixNano":"1581452773000001459","timeUnixNano":"1581452773000001459","asInt":"456"}],"aggregationTemporality":2,"isMonotonic":true}}]}]}]} +{"resourceMetrics":[{"resource":{"attributes":[{"key":"resource-attr","value":{"stringValue":"resource-attr-val-1"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"counter-int","unit":"1","sum":{"dataPoints":[{"attributes":[{"key":"label-1","value":{"stringValue":"label-value-1"}}],"startTimeUnixNano":"1581452773000002346","timeUnixNano":"1581452773000002346","asInt":"121"},{"attributes":[{"key":"label-2","value":{"stringValue":"label-value-2"}}],"startTimeUnixNano":"1581452773000002346","timeUnixNano":"1581452773000002346","asInt":"456"}],"aggregationTemporality":2,"isMonotonic":true}},{"name":"counter-int","unit":"1","sum":{"dataPoints":[{"attributes":[{"key":"label-1","value":{"stringValue":"label-value-1"}}],"startTimeUnixNano":"1581452773000002346","timeUnixNano":"1581452773000002346","asInt":"123"},{"attributes":[{"key":"label-2","value":{"stringValue":"label-value-2"}}],"startTimeUnixNano":"1581452772000000321","timeUnixNano":"1581452773000000789","asInt":"456"}],"aggregationTemporality":2,"isMonotonic":true}}]}]}]} +{"resourceMetrics":[{"resource":{"attributes":[{"key":"resource-attr","value":{"stringValue":"resource-attr-val-1"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"counter-int","unit":"1","sum":{"dataPoints":[{"attributes":[{"key":"label-1","value":{"stringValue":"label-value-1"}}],"startTimeUnixNano":"1581452773000007891","timeUnixNano":"1581452773000007891","asInt":"125"},{"attributes":[{"key":"label-2","value":{"stringValue":"label-value-2"}}],"startTimeUnixNano":"1581452772000000321","timeUnixNano":"1581452773000007891","asInt":"456"}],"aggregationTemporality":2,"isMonotonic":true}},{"name":"counter-int","unit":"1","sum":{"dataPoints":[{"attributes":[{"key":"label-1","value":{"stringValue":"label-value-1"}}],"startTimeUnixNano":"1581452773000007891","timeUnixNano":"1581452773000007891","asInt":"123"},{"attributes":[{"key":"label-2","value":{"stringValue":"label-value-2"}}],"startTimeUnixNano":"1581452772000000321","timeUnixNano":"1581452773000007891","asInt":"456"}],"aggregationTemporality":2,"isMonotonic":true}}]}]}]} + diff --git a/internal/loadgen/events/otlp-traces-0.0.0.jsonl b/internal/loadgen/events/otlp-traces-0.0.0.jsonl new file mode 100644 index 0000000..76774ba --- /dev/null +++ b/internal/loadgen/events/otlp-traces-0.0.0.jsonl @@ -0,0 +1,5 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"resource-attr","value":{"stringValue":"resource-attr-val-1"}}]},"scopeSpans":[{"scope":{},"spans":[{"traceId":"","spanId":"","parentSpanId":"","name":"operationA","startTimeUnixNano":"1581452772000000321","endTimeUnixNano":"1581452773000000789","droppedAttributesCount":1,"events":[{"timeUnixNano":"1581452773000000123","name":"event-with-attr","attributes":[{"key":"span-event-attr","value":{"stringValue":"span-event-attr-val"}}],"droppedAttributesCount":2},{"timeUnixNano":"1581452773000000123","name":"event","droppedAttributesCount":2}],"droppedEventsCount":1,"status":{"message":"status-cancelled","code":2}},{"traceId":"","spanId":"","parentSpanId":"","name":"operationB","startTimeUnixNano":"1581452772000000321","endTimeUnixNano":"1581452773000000789","links":[{"traceId":"","spanId":"","attributes":[{"key":"span-link-attr","value":{"stringValue":"span-link-attr-val"}}],"droppedAttributesCount":4},{"traceId":"","spanId":"","droppedAttributesCount":1}],"droppedLinksCount":3,"status":{}}]}]}]} +{"resourceSpans":[{"resource":{"attributes":[{"key":"resource-attr","value":{"stringValue":"resource-attr-val-1"}}]},"scopeSpans":[{"scope":{},"spans":[{"traceId":"","spanId":"","parentSpanId":"","name":"operationA","startTimeUnixNano":"1581452772000000321","endTimeUnixNano":"1581452773000000789","droppedAttributesCount":1,"events":[{"timeUnixNano":"1581452773000000424","name":"event-with-attr","attributes":[{"key":"span-event-attr","value":{"stringValue":"span-event-attr-val"}}],"droppedAttributesCount":2},{"timeUnixNano":"1581452773000000424","name":"event","droppedAttributesCount":2}],"droppedEventsCount":1,"status":{"message":"status-cancelled","code":2}},{"traceId":"","spanId":"","parentSpanId":"","name":"operationB","startTimeUnixNano":"1581452772000000343","endTimeUnixNano":"1581452773000001089","links":[{"traceId":"","spanId":"","attributes":[{"key":"span-link-attr","value":{"stringValue":"span-link-attr-val"}}],"droppedAttributesCount":3},{"traceId":"","spanId":"","droppedAttributesCount":4}],"droppedLinksCount":2,"status":{}}]}]}]} +{"resourceSpans":[{"resource":{"attributes":[{"key":"resource-attr","value":{"stringValue":"resource-attr-val-1"}}]},"scopeSpans":[{"scope":{},"spans":[{"traceId":"","spanId":"","parentSpanId":"","name":"operationA","startTimeUnixNano":"1581452772000000321","endTimeUnixNano":"1581452773000000789","droppedAttributesCount":1,"events":[{"timeUnixNano":"1581452773000000826","name":"event-with-attr","attributes":[{"key":"span-event-attr","value":{"stringValue":"span-event-attr-val"}}],"droppedAttributesCount":2},{"timeUnixNano":"1581452773000000826","name":"event","droppedAttributesCount":2}],"droppedEventsCount":1,"status":{"message":"status-cancelled","code":2}},{"traceId":"","spanId":"","parentSpanId":"","name":"operationB","startTimeUnixNano":"1581452772000200521","endTimeUnixNano":"1581452773000004789","links":[{"traceId":"","spanId":"","attributes":[{"key":"span-link-attr","value":{"stringValue":"span-link-attr-val"}}],"droppedAttributesCount":5},{"traceId":"","spanId":"","droppedAttributesCount":2}],"droppedLinksCount":3,"status":{}}]}]}]} +{"resourceSpans":[{"resource":{"attributes":[{"key":"resource-attr","value":{"stringValue":"resource-attr-val-1"}}]},"scopeSpans":[{"scope":{},"spans":[{"traceId":"","spanId":"","parentSpanId":"","name":"operationA","startTimeUnixNano":"1581452772000000321","endTimeUnixNano":"1581452773000000789","droppedAttributesCount":1,"events":[{"timeUnixNano":"1581452773000010925","name":"event-with-attr","attributes":[{"key":"span-event-attr","value":{"stringValue":"span-event-attr-val"}}],"droppedAttributesCount":2},{"timeUnixNano":"1581452773000010925","name":"event","droppedAttributesCount":2}],"droppedEventsCount":1,"status":{"message":"status-cancelled","code":2}},{"traceId":"","spanId":"","parentSpanId":"","name":"operationB","startTimeUnixNano":"1581452772000011821","endTimeUnixNano":"1581452772000012924","links":[{"traceId":"","spanId":"","attributes":[{"key":"span-link-attr","value":{"stringValue":"span-link-attr-val"}}],"droppedAttributesCount":2},{"traceId":"","spanId":"","droppedAttributesCount":2}],"droppedLinksCount":5,"status":{}}]}]}]} + diff --git a/internal/soaktest/runner.go b/internal/soaktest/runner.go index 47d9d4b..b839960 100644 --- a/internal/soaktest/runner.go +++ b/internal/soaktest/runner.go @@ -119,7 +119,11 @@ func getHandlerParams(runnerConfig *RunnerConfig, config ScenarioConfig) (loadge if config.AgentName == "" { config.AgentName = "apm-" } - path := config.AgentName + "*.ndjson" + extension := ".ndjson" + if strings.HasPrefix(config.AgentName, "otlp-") { + extension = ".jsonl" + } + path := config.AgentName + "*" + extension var params loadgen.EventHandlerParams if config.Headers == nil {