From 8fd43a836e44a8dfbf5648139f8561c4e380f510 Mon Sep 17 00:00:00 2001 From: Alexandre Lamarre Date: Thu, 31 Aug 2023 11:52:18 -0400 Subject: [PATCH 1/7] moving some stuff WIP slo testability WIP breaking everything wip integration testing the great slo purge update mocks Remove old/unused test code to fix integration test failures fix tests enhance SLO validation lint fixes window improvements WIP MWMB alerts fix filters metrics store mocked tests fix lint fix build basic prometheus SLO integration tests metric SLO discovery integration test add list datasource API prometheus unmarshalling enhancements add MWMB Alerts to prometheus generator update interface update tests consistently name imports stuff increase cortex reachable timeout --- ...90-0f9b-446a-b936-7da6b853de11-alerts.yaml | 37 + ...-0f9b-446a-b936-7da6b853de11-metadata.yaml | 49 + ...0f9b-446a-b936-7da6b853de11-recording.yaml | 52 + internal/cmd/testenv/main.go | 1 - pkg/apis/core/v1/extensions.go | 9 + .../apis/slo/v1/extensions.go | 12 +- .../apis/slo => pkg/apis/slo/v1}/slo.pb.go | 1126 +- .../apis/slo => pkg/apis/slo/v1}/slo.proto | 15 +- .../slo => pkg/apis/slo/v1}/slo_grpc.pb.go | 69 +- .../apis/slo => pkg/apis/slo/v1}/validate.go | 8 +- pkg/metrics/compat/compat_suite_test.go | 14 + pkg/metrics/compat/unmarshal.go | 67 +- pkg/metrics/compat/unmarshal_test.go | 198 + pkg/slo/backend/backend_suite_test.go | 14 + pkg/slo/backend/event_test.go | 122 + pkg/slo/backend/metrics/backend.go | 88 + pkg/slo/backend/metrics/backend_provider.go | 184 + pkg/slo/backend/metrics/backend_test.go | 96 + pkg/slo/backend/metrics/cortex.go | 9 + pkg/slo/backend/metrics/filter_test.go | 73 + pkg/slo/backend/metrics/manager.go | 375 + pkg/slo/backend/metrics/manager_test.go | 162 + .../metrics}/metricgroups/compute.yaml | 0 .../metrics}/metricgroups/go_lang.yaml | 0 .../metrics}/metricgroups/java_lang.yaml | 0 .../metrics}/metricgroups/network.yaml | 0 .../backend/metrics}/metricgroups/other.yaml | 0 pkg/slo/backend/metrics/metrics_suite_test.go | 14 + pkg/slo/backend/metrics/prometheus.go | 584 + pkg/slo/backend/metrics/prometheus_test.go | 990 + .../metrics}/servicegroups/kubernetes.yaml | 16 +- .../slo => pkg/slo/backend/metrics}/window.go | 124 +- pkg/slo/backend/metrics/window_test.go | 51 + pkg/slo/backend/service.go | 13 + pkg/slo/backend/slo.go | 308 + pkg/slo/backend/slo_test.go | 3 + pkg/slo/backend/store.go | 24 + pkg/slo/backend/util.go | 96 + pkg/slo/backend/util_test.go | 60 + pkg/slo/query/query_test.go | 2 +- pkg/test/mock/cortexadmin/cortexadmin.go | 633 + pkg/test/mock/mockgen.yaml | 13 + pkg/test/mock/slo/service.go | 81 + pkg/test/mock/slo/store.go | 143 + plugins/alerting/pkg/alerting/plugin.go | 24 + .../alerting/pkg/alerting/slo/v1/component.go | 96 + .../alerting/pkg/alerting/slo/v1/server.go | 311 + plugins/alerting/pkg/alerting/system.go | 51 + plugins/slo/apis/slo/compat.go | 11 - plugins/slo/main.go | 16 - plugins/slo/pkg/slo/api.go | 266 - plugins/slo/pkg/slo/cortex_info.go | 224 - plugins/slo/pkg/slo/filters.go | 165 - plugins/slo/pkg/slo/filters_test.go | 68 - plugins/slo/pkg/slo/impl.go | 422 - plugins/slo/pkg/slo/interfaces.go | 92 - plugins/slo/pkg/slo/plugin.go | 60 - plugins/slo/pkg/slo/simple.go | 859 - plugins/slo/pkg/slo/slo_suite_test.go | 16 - plugins/slo/pkg/slo/slo_test.go | 826 - plugins/slo/pkg/slo/system.go | 48 - plugins/slo/pkg/slo/util.go | 68 - plugins/slo/test/test_plugin.go | 15 - test/integration/managementapi_cert_test.go | 56 - test/plugins/alerting/output.log | 20043 ++++++++++++++++ test/plugins/alerting/slo_test.go | 564 + test/plugins/alerting/testlog.txt | 5437 +++++ test/plugins/metrics/slo_test.go | 567 + test/plugins/slo/slo_suite_test.go | 17 - test/plugins/slo/slo_test.go | 688 - test/plugins/slo/test_helpers_test.go | 95 - 71 files changed, 32411 insertions(+), 4629 deletions(-) create mode 100644 dc3d8690-0f9b-446a-b936-7da6b853de11-alerts.yaml create mode 100644 dc3d8690-0f9b-446a-b936-7da6b853de11-metadata.yaml create mode 100644 dc3d8690-0f9b-446a-b936-7da6b853de11-recording.yaml rename plugins/slo/apis/slo/extension.go => pkg/apis/slo/v1/extensions.go (65%) rename {plugins/slo/apis/slo => pkg/apis/slo/v1}/slo.pb.go (51%) rename {plugins/slo/apis/slo => pkg/apis/slo/v1}/slo.proto (94%) rename {plugins/slo/apis/slo => pkg/apis/slo/v1}/slo_grpc.pb.go (87%) rename {plugins/slo/apis/slo => pkg/apis/slo/v1}/validate.go (92%) create mode 100644 pkg/metrics/compat/compat_suite_test.go create mode 100644 pkg/metrics/compat/unmarshal_test.go create mode 100644 pkg/slo/backend/backend_suite_test.go create mode 100644 pkg/slo/backend/event_test.go create mode 100644 pkg/slo/backend/metrics/backend.go create mode 100644 pkg/slo/backend/metrics/backend_provider.go create mode 100644 pkg/slo/backend/metrics/backend_test.go create mode 100644 pkg/slo/backend/metrics/cortex.go create mode 100644 pkg/slo/backend/metrics/filter_test.go create mode 100644 pkg/slo/backend/metrics/manager.go create mode 100644 pkg/slo/backend/metrics/manager_test.go rename {plugins/slo/pkg/slo => pkg/slo/backend/metrics}/metricgroups/compute.yaml (100%) rename {plugins/slo/pkg/slo => pkg/slo/backend/metrics}/metricgroups/go_lang.yaml (100%) rename {plugins/slo/pkg/slo => pkg/slo/backend/metrics}/metricgroups/java_lang.yaml (100%) rename {plugins/slo/pkg/slo => pkg/slo/backend/metrics}/metricgroups/network.yaml (100%) rename {plugins/slo/pkg/slo => pkg/slo/backend/metrics}/metricgroups/other.yaml (100%) create mode 100644 pkg/slo/backend/metrics/metrics_suite_test.go create mode 100644 pkg/slo/backend/metrics/prometheus.go create mode 100644 pkg/slo/backend/metrics/prometheus_test.go rename {plugins/slo/pkg/slo => pkg/slo/backend/metrics}/servicegroups/kubernetes.yaml (87%) rename {plugins/slo/pkg/slo => pkg/slo/backend/metrics}/window.go (53%) create mode 100644 pkg/slo/backend/metrics/window_test.go create mode 100644 pkg/slo/backend/service.go create mode 100644 pkg/slo/backend/slo.go create mode 100644 pkg/slo/backend/slo_test.go create mode 100644 pkg/slo/backend/store.go create mode 100644 pkg/slo/backend/util.go create mode 100644 pkg/slo/backend/util_test.go create mode 100644 pkg/test/mock/cortexadmin/cortexadmin.go create mode 100644 pkg/test/mock/slo/service.go create mode 100644 pkg/test/mock/slo/store.go create mode 100644 plugins/alerting/pkg/alerting/slo/v1/component.go create mode 100644 plugins/alerting/pkg/alerting/slo/v1/server.go delete mode 100644 plugins/slo/apis/slo/compat.go delete mode 100644 plugins/slo/main.go delete mode 100644 plugins/slo/pkg/slo/api.go delete mode 100644 plugins/slo/pkg/slo/cortex_info.go delete mode 100644 plugins/slo/pkg/slo/filters.go delete mode 100644 plugins/slo/pkg/slo/filters_test.go delete mode 100644 plugins/slo/pkg/slo/impl.go delete mode 100644 plugins/slo/pkg/slo/interfaces.go delete mode 100644 plugins/slo/pkg/slo/plugin.go delete mode 100644 plugins/slo/pkg/slo/simple.go delete mode 100644 plugins/slo/pkg/slo/slo_suite_test.go delete mode 100644 plugins/slo/pkg/slo/slo_test.go delete mode 100644 plugins/slo/pkg/slo/system.go delete mode 100644 plugins/slo/pkg/slo/util.go delete mode 100644 plugins/slo/test/test_plugin.go delete mode 100644 test/integration/managementapi_cert_test.go create mode 100644 test/plugins/alerting/output.log create mode 100644 test/plugins/alerting/slo_test.go create mode 100644 test/plugins/alerting/testlog.txt create mode 100644 test/plugins/metrics/slo_test.go delete mode 100644 test/plugins/slo/slo_suite_test.go delete mode 100644 test/plugins/slo/slo_test.go delete mode 100644 test/plugins/slo/test_helpers_test.go diff --git a/dc3d8690-0f9b-446a-b936-7da6b853de11-alerts.yaml b/dc3d8690-0f9b-446a-b936-7da6b853de11-alerts.yaml new file mode 100644 index 0000000000..1a3aa01244 --- /dev/null +++ b/dc3d8690-0f9b-446a-b936-7da6b853de11-alerts.yaml @@ -0,0 +1,37 @@ +name: dc3d8690-0f9b-446a-b936-7da6b853de11-alerts +interval: 1m +rules: + - record: slo:alert:ticket_window + expr: (max(slo:sli_error:ratio_rate5m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000347222 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate30m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000115741 * 5.000000000)) without (slo_window)) or (max(slo:sli_error:ratio_rate2h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000347222 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate6h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000115741 * 5.000000000)) without (slo_window)) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_severity: ticket + - record: slo:alert:page_window + expr: (max(slo:sli_error:ratio_rate5m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.001666667 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate30m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000694444 * 5.000000000)) without (slo_window)) or (max(slo:sli_error:ratio_rate2h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.001666667 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate6h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000694444 * 5.000000000)) without (slo_window)) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_severity: page + - alert: dc3d8690-0f9b-446a-b936-7da6b853de11-page-alerts + expr: (max(slo:sli_error:ratio_rate5m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.001666667 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate30m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000694444 * 5.000000000)) without (slo_window)) or (max(slo:sli_error:ratio_rate2h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.001666667 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate6h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000694444 * 5.000000000)) without (slo_window)) + labels: + opni_uuid: dc3d8690-0f9b-446a-b936-7da6b853de11-page-alerts + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_severity: page + annotations: + opni_uuid: dc3d8690-0f9b-446a-b936-7da6b853de11-page-alerts + - alert: dc3d8690-0f9b-446a-b936-7da6b853de11-ticket-alerts + expr: (max(slo:sli_error:ratio_rate5m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000347222 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate30m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000115741 * 5.000000000)) without (slo_window)) or (max(slo:sli_error:ratio_rate2h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000347222 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate6h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000115741 * 5.000000000)) without (slo_window)) + labels: + opni_uuid: dc3d8690-0f9b-446a-b936-7da6b853de11-ticket-alerts + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_severity: ticket + annotations: + opni_uuid: dc3d8690-0f9b-446a-b936-7da6b853de11-ticket-alerts diff --git a/dc3d8690-0f9b-446a-b936-7da6b853de11-metadata.yaml b/dc3d8690-0f9b-446a-b936-7da6b853de11-metadata.yaml new file mode 100644 index 0000000000..6336f7b6f1 --- /dev/null +++ b/dc3d8690-0f9b-446a-b936-7da6b853de11-metadata.yaml @@ -0,0 +1,49 @@ +name: dc3d8690-0f9b-446a-b936-7da6b853de11-metadata +interval: 1m +rules: + - record: slo:objective:ratio + expr: vector(0.950000000) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + - record: slo:error_budget:ratio + expr: vector(1-0.950000000) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + - record: slo:time_period:days + expr: vector(7) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + - record: slo:current_burn_rate:ratio + expr: |- + slo:sli_error:ratio_rate5m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} / on(slo_opni_id, slo_opni_service, slo_opni_name) group_left + slo:error_budget:ratio{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + - record: slo:period_burn_rate:ratio + expr: |- + slo:sli_error:ratio_rate7d{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} / on(slo_opni_id, slo_opni_service, slo_opni_name) group_left + slo:error_budget:ratio{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + - record: slo:period_error_budget_remaining:ratio + expr: 1 - slo:period_burn_rate:ratio{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + - record: opni_slo_info + expr: vector(1) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway diff --git a/dc3d8690-0f9b-446a-b936-7da6b853de11-recording.yaml b/dc3d8690-0f9b-446a-b936-7da6b853de11-recording.yaml new file mode 100644 index 0000000000..751353a494 --- /dev/null +++ b/dc3d8690-0f9b-446a-b936-7da6b853de11-recording.yaml @@ -0,0 +1,52 @@ +name: dc3d8690-0f9b-446a-b936-7da6b853de11-recording +interval: 1m +rules: + - record: slo:sli_error:ratio_rate5m + expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[5m])))/(sum(rate(process_max_fds{job="opni-gateway"}[5m])))) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_window: 5m + - record: slo:sli_error:ratio_rate30m + expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[30m])))/(sum(rate(process_max_fds{job="opni-gateway"}[30m])))) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_window: 30m + - record: slo:sli_error:ratio_rate1h + expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[1h])))/(sum(rate(process_max_fds{job="opni-gateway"}[1h])))) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_window: 1h + - record: slo:sli_error:ratio_rate2h + expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[2h])))/(sum(rate(process_max_fds{job="opni-gateway"}[2h])))) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_window: 2h + - record: slo:sli_error:ratio_rate6h + expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[6h])))/(sum(rate(process_max_fds{job="opni-gateway"}[6h])))) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_window: 6h + - record: slo:sli_error:ratio_rate1d + expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[1d])))/(sum(rate(process_max_fds{job="opni-gateway"}[1d])))) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_window: 1d + - record: slo:sli_error:ratio_rate7d + expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[7d])))/(sum(rate(process_max_fds{job="opni-gateway"}[7d])))) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_window: 7d diff --git a/internal/cmd/testenv/main.go b/internal/cmd/testenv/main.go index 82a670f3dc..e4222159d2 100644 --- a/internal/cmd/testenv/main.go +++ b/internal/cmd/testenv/main.go @@ -48,7 +48,6 @@ import ( _ "github.com/rancher/opni/plugins/alerting/test" _ "github.com/rancher/opni/plugins/logging/test" _ "github.com/rancher/opni/plugins/metrics/test" - _ "github.com/rancher/opni/plugins/slo/test" ) func main() { diff --git a/pkg/apis/core/v1/extensions.go b/pkg/apis/core/v1/extensions.go index aa9fd6b413..ce30f387f9 100644 --- a/pkg/apis/core/v1/extensions.go +++ b/pkg/apis/core/v1/extensions.go @@ -9,6 +9,15 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) +func (c *ClusterList) Has(clusterId string) bool { + for _, cluster := range c.Items { + if cluster.Id == clusterId { + return true + } + } + return false +} + func (s *HealthStatus) Summary() string { if s.Status == nil || s.Health == nil { return "Unknown" diff --git a/plugins/slo/apis/slo/extension.go b/pkg/apis/slo/v1/extensions.go similarity index 65% rename from plugins/slo/apis/slo/extension.go rename to pkg/apis/slo/v1/extensions.go index 44b95f7aee..363075745a 100644 --- a/plugins/slo/apis/slo/extension.go +++ b/pkg/apis/slo/v1/extensions.go @@ -1,4 +1,4 @@ -package slo +package v1 import ( "slices" @@ -19,5 +19,13 @@ func (s *ServiceList) ContainsId(id string) bool { for _, svc := range s.Items { svcIds = append(svcIds, svc.ServiceId) } - return !slices.Contains(svcIds, id) + return slices.Contains(svcIds, id) +} + +func (e *EventList) ContainsId(id string) bool { + var eventIds []string + for _, event := range e.Items { + eventIds = append(eventIds, event.Key) + } + return slices.Contains(eventIds, id) } diff --git a/plugins/slo/apis/slo/slo.pb.go b/pkg/apis/slo/v1/slo.pb.go similarity index 51% rename from plugins/slo/apis/slo/slo.pb.go rename to pkg/apis/slo/v1/slo.pb.go index 63f48f0b72..f6572ebd3e 100644 --- a/plugins/slo/apis/slo/slo.pb.go +++ b/pkg/apis/slo/v1/slo.pb.go @@ -2,9 +2,9 @@ // versions: // protoc-gen-go v1.30.0-devel // protoc v1.0.0 -// source: github.com/rancher/opni/plugins/slo/apis/slo/slo.proto +// source: github.com/rancher/opni/pkg/apis/slo/v1/slo.proto -package slo +package v1 import ( v11 "github.com/rancher/opni/pkg/apis/alerting/v1" @@ -76,11 +76,11 @@ func (x SLOStatusState) String() string { } func (SLOStatusState) Descriptor() protoreflect.EnumDescriptor { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_enumTypes[0].Descriptor() + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_enumTypes[0].Descriptor() } func (SLOStatusState) Type() protoreflect.EnumType { - return &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_enumTypes[0] + return &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_enumTypes[0] } func (x SLOStatusState) Number() protoreflect.EnumNumber { @@ -89,7 +89,54 @@ func (x SLOStatusState) Number() protoreflect.EnumNumber { // Deprecated: Use SLOStatusState.Descriptor instead. func (SLOStatusState) EnumDescriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{0} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{0} +} + +type DatasourceInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Items []string `protobuf:"bytes,1,rep,name=items,proto3" json:"items,omitempty"` +} + +func (x *DatasourceInfo) Reset() { + *x = DatasourceInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DatasourceInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DatasourceInfo) ProtoMessage() {} + +func (x *DatasourceInfo) ProtoReflect() protoreflect.Message { + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DatasourceInfo.ProtoReflect.Descriptor instead. +func (*DatasourceInfo) Descriptor() ([]byte, []int) { + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{0} +} + +func (x *DatasourceInfo) GetItems() []string { + if x != nil { + return x.Items + } + return nil } type MultiClusterSLO struct { @@ -104,7 +151,7 @@ type MultiClusterSLO struct { func (x *MultiClusterSLO) Reset() { *x = MultiClusterSLO{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[0] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -117,7 +164,7 @@ func (x *MultiClusterSLO) String() string { func (*MultiClusterSLO) ProtoMessage() {} func (x *MultiClusterSLO) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[0] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -130,7 +177,7 @@ func (x *MultiClusterSLO) ProtoReflect() protoreflect.Message { // Deprecated: Use MultiClusterSLO.ProtoReflect.Descriptor instead. func (*MultiClusterSLO) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{0} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{1} } func (x *MultiClusterSLO) GetCloneId() *v1.Reference { @@ -158,7 +205,7 @@ type MultiClusterFailures struct { func (x *MultiClusterFailures) Reset() { *x = MultiClusterFailures{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[1] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -171,7 +218,7 @@ func (x *MultiClusterFailures) String() string { func (*MultiClusterFailures) ProtoMessage() {} func (x *MultiClusterFailures) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[1] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -184,7 +231,7 @@ func (x *MultiClusterFailures) ProtoReflect() protoreflect.Message { // Deprecated: Use MultiClusterFailures.ProtoReflect.Descriptor instead. func (*MultiClusterFailures) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{1} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{2} } func (x *MultiClusterFailures) GetFailures() []string { @@ -208,7 +255,7 @@ type ListEventsRequest struct { func (x *ListEventsRequest) Reset() { *x = ListEventsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[2] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -221,7 +268,7 @@ func (x *ListEventsRequest) String() string { func (*ListEventsRequest) ProtoMessage() {} func (x *ListEventsRequest) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[2] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -234,7 +281,7 @@ func (x *ListEventsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListEventsRequest.ProtoReflect.Descriptor instead. func (*ListEventsRequest) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{2} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{3} } func (x *ListEventsRequest) GetDatasource() string { @@ -278,7 +325,7 @@ type ListMetricsRequest struct { func (x *ListMetricsRequest) Reset() { *x = ListMetricsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[3] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -291,7 +338,7 @@ func (x *ListMetricsRequest) String() string { func (*ListMetricsRequest) ProtoMessage() {} func (x *ListMetricsRequest) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[3] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -304,7 +351,7 @@ func (x *ListMetricsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListMetricsRequest.ProtoReflect.Descriptor instead. func (*ListMetricsRequest) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{3} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{4} } func (x *ListMetricsRequest) GetDatasource() string { @@ -340,7 +387,7 @@ type ListServicesRequest struct { func (x *ListServicesRequest) Reset() { *x = ListServicesRequest{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[4] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -353,7 +400,7 @@ func (x *ListServicesRequest) String() string { func (*ListServicesRequest) ProtoMessage() {} func (x *ListServicesRequest) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[4] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -366,7 +413,7 @@ func (x *ListServicesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListServicesRequest.ProtoReflect.Descriptor instead. func (*ListServicesRequest) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{4} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{5} } func (x *ListServicesRequest) GetDatasource() string { @@ -394,7 +441,7 @@ type EventList struct { func (x *EventList) Reset() { *x = EventList{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[5] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -407,7 +454,7 @@ func (x *EventList) String() string { func (*EventList) ProtoMessage() {} func (x *EventList) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[5] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -420,7 +467,7 @@ func (x *EventList) ProtoReflect() protoreflect.Message { // Deprecated: Use EventList.ProtoReflect.Descriptor instead. func (*EventList) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{5} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{6} } func (x *EventList) GetItems() []*Event { @@ -442,7 +489,7 @@ type Event struct { func (x *Event) Reset() { *x = Event{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[6] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -455,7 +502,7 @@ func (x *Event) String() string { func (*Event) ProtoMessage() {} func (x *Event) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[6] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -468,7 +515,7 @@ func (x *Event) ProtoReflect() protoreflect.Message { // Deprecated: Use Event.ProtoReflect.Descriptor instead. func (*Event) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{6} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{7} } func (x *Event) GetKey() string { @@ -500,7 +547,7 @@ type ServiceInfo struct { func (x *ServiceInfo) Reset() { *x = ServiceInfo{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[7] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -513,7 +560,7 @@ func (x *ServiceInfo) String() string { func (*ServiceInfo) ProtoMessage() {} func (x *ServiceInfo) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[7] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -526,7 +573,7 @@ func (x *ServiceInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use ServiceInfo.ProtoReflect.Descriptor instead. func (*ServiceInfo) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{7} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{8} } func (x *ServiceInfo) GetJobId() string { @@ -576,7 +623,7 @@ type Service struct { func (x *Service) Reset() { *x = Service{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[8] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -589,7 +636,7 @@ func (x *Service) String() string { func (*Service) ProtoMessage() {} func (x *Service) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[8] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -602,7 +649,7 @@ func (x *Service) ProtoReflect() protoreflect.Message { // Deprecated: Use Service.ProtoReflect.Descriptor instead. func (*Service) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{8} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{9} } func (x *Service) GetServiceId() string { @@ -630,7 +677,7 @@ type ServiceList struct { func (x *ServiceList) Reset() { *x = ServiceList{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[9] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -643,7 +690,7 @@ func (x *ServiceList) String() string { func (*ServiceList) ProtoMessage() {} func (x *ServiceList) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[9] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -656,7 +703,7 @@ func (x *ServiceList) ProtoReflect() protoreflect.Message { // Deprecated: Use ServiceList.ProtoReflect.Descriptor instead. func (*ServiceList) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{9} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{10} } func (x *ServiceList) GetItems() []*Service { @@ -677,7 +724,7 @@ type Label struct { func (x *Label) Reset() { *x = Label{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[10] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -690,7 +737,7 @@ func (x *Label) String() string { func (*Label) ProtoMessage() {} func (x *Label) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[10] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -703,7 +750,7 @@ func (x *Label) ProtoReflect() protoreflect.Message { // Deprecated: Use Label.ProtoReflect.Descriptor instead. func (*Label) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{10} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{11} } func (x *Label) GetName() string { @@ -733,7 +780,7 @@ type Alert struct { func (x *Alert) Reset() { *x = Alert{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[11] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -746,7 +793,7 @@ func (x *Alert) String() string { func (*Alert) ProtoMessage() {} func (x *Alert) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[11] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -759,7 +806,7 @@ func (x *Alert) ProtoReflect() protoreflect.Message { // Deprecated: Use Alert.ProtoReflect.Descriptor instead. func (*Alert) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{11} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{12} } func (x *Alert) GetName() string { @@ -855,7 +902,7 @@ type ServiceLevelObjective struct { func (x *ServiceLevelObjective) Reset() { *x = ServiceLevelObjective{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[12] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -868,7 +915,7 @@ func (x *ServiceLevelObjective) String() string { func (*ServiceLevelObjective) ProtoMessage() {} func (x *ServiceLevelObjective) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[12] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -881,7 +928,7 @@ func (x *ServiceLevelObjective) ProtoReflect() protoreflect.Message { // Deprecated: Use ServiceLevelObjective.ProtoReflect.Descriptor instead. func (*ServiceLevelObjective) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{12} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{13} } func (x *ServiceLevelObjective) GetName() string { @@ -986,7 +1033,7 @@ type CreateSLORequest struct { func (x *CreateSLORequest) Reset() { *x = CreateSLORequest{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[13] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -999,7 +1046,7 @@ func (x *CreateSLORequest) String() string { func (*CreateSLORequest) ProtoMessage() {} func (x *CreateSLORequest) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[13] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1012,7 +1059,7 @@ func (x *CreateSLORequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateSLORequest.ProtoReflect.Descriptor instead. func (*CreateSLORequest) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{13} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{14} } func (x *CreateSLORequest) GetSlo() *ServiceLevelObjective { @@ -1033,7 +1080,7 @@ type EventPairList struct { func (x *EventPairList) Reset() { *x = EventPairList{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[14] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1046,7 +1093,7 @@ func (x *EventPairList) String() string { func (*EventPairList) ProtoMessage() {} func (x *EventPairList) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[14] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1059,7 +1106,7 @@ func (x *EventPairList) ProtoReflect() protoreflect.Message { // Deprecated: Use EventPairList.ProtoReflect.Descriptor instead. func (*EventPairList) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{14} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{15} } func (x *EventPairList) GetItems() []*EventPair { @@ -1081,7 +1128,7 @@ type EventPair struct { func (x *EventPair) Reset() { *x = EventPair{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[15] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1094,7 +1141,7 @@ func (x *EventPair) String() string { func (*EventPair) ProtoMessage() {} func (x *EventPair) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[15] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1107,7 +1154,7 @@ func (x *EventPair) ProtoReflect() protoreflect.Message { // Deprecated: Use EventPair.ProtoReflect.Descriptor instead. func (*EventPair) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{15} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{16} } func (x *EventPair) GetKey() string { @@ -1137,7 +1184,7 @@ type SLOData struct { func (x *SLOData) Reset() { *x = SLOData{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[16] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1150,7 +1197,7 @@ func (x *SLOData) String() string { func (*SLOData) ProtoMessage() {} func (x *SLOData) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[16] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1163,7 +1210,7 @@ func (x *SLOData) ProtoReflect() protoreflect.Message { // Deprecated: Use SLOData.ProtoReflect.Descriptor instead. func (*SLOData) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{16} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{17} } func (x *SLOData) GetId() string { @@ -1198,7 +1245,7 @@ type Target struct { func (x *Target) Reset() { *x = Target{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[17] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1211,7 +1258,7 @@ func (x *Target) String() string { func (*Target) ProtoMessage() {} func (x *Target) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[17] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[18] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1224,7 +1271,7 @@ func (x *Target) ProtoReflect() protoreflect.Message { // Deprecated: Use Target.ProtoReflect.Descriptor instead. func (*Target) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{17} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{18} } func (x *Target) GetValue() float64 { @@ -1246,7 +1293,7 @@ type Metric struct { func (x *Metric) Reset() { *x = Metric{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[18] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1259,7 +1306,7 @@ func (x *Metric) String() string { func (*Metric) ProtoMessage() {} func (x *Metric) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[18] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1272,7 +1319,7 @@ func (x *Metric) ProtoReflect() protoreflect.Message { // Deprecated: Use Metric.ProtoReflect.Descriptor instead. func (*Metric) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{18} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{19} } func (x *Metric) GetId() string { @@ -1302,7 +1349,7 @@ type MetricMetadata struct { func (x *MetricMetadata) Reset() { *x = MetricMetadata{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[19] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1315,7 +1362,7 @@ func (x *MetricMetadata) String() string { func (*MetricMetadata) ProtoMessage() {} func (x *MetricMetadata) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[19] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[20] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1328,7 +1375,7 @@ func (x *MetricMetadata) ProtoReflect() protoreflect.Message { // Deprecated: Use MetricMetadata.ProtoReflect.Descriptor instead. func (*MetricMetadata) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{19} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{20} } func (x *MetricMetadata) GetDescription() string { @@ -1363,7 +1410,7 @@ type MetricList struct { func (x *MetricList) Reset() { *x = MetricList{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[20] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1376,7 +1423,7 @@ func (x *MetricList) String() string { func (*MetricList) ProtoMessage() {} func (x *MetricList) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[20] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[21] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1389,7 +1436,7 @@ func (x *MetricList) ProtoReflect() protoreflect.Message { // Deprecated: Use MetricList.ProtoReflect.Descriptor instead. func (*MetricList) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{20} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{21} } func (x *MetricList) GetItems() []*Metric { @@ -1410,7 +1457,7 @@ type MetricGroupList struct { func (x *MetricGroupList) Reset() { *x = MetricGroupList{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[21] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1423,7 +1470,7 @@ func (x *MetricGroupList) String() string { func (*MetricGroupList) ProtoMessage() {} func (x *MetricGroupList) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[21] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1436,7 +1483,7 @@ func (x *MetricGroupList) ProtoReflect() protoreflect.Message { // Deprecated: Use MetricGroupList.ProtoReflect.Descriptor instead. func (*MetricGroupList) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{21} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{22} } func (x *MetricGroupList) GetGroupNameToMetrics() map[string]*MetricList { @@ -1457,7 +1504,7 @@ type ServiceLevelObjectiveList struct { func (x *ServiceLevelObjectiveList) Reset() { *x = ServiceLevelObjectiveList{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[22] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1470,7 +1517,7 @@ func (x *ServiceLevelObjectiveList) String() string { func (*ServiceLevelObjectiveList) ProtoMessage() {} func (x *ServiceLevelObjectiveList) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[22] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[23] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1483,7 +1530,7 @@ func (x *ServiceLevelObjectiveList) ProtoReflect() protoreflect.Message { // Deprecated: Use ServiceLevelObjectiveList.ProtoReflect.Descriptor instead. func (*ServiceLevelObjectiveList) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{22} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{23} } func (x *ServiceLevelObjectiveList) GetItems() []*SLOData { @@ -1504,7 +1551,7 @@ type SLOStatus struct { func (x *SLOStatus) Reset() { *x = SLOStatus{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[23] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1517,7 +1564,7 @@ func (x *SLOStatus) String() string { func (*SLOStatus) ProtoMessage() {} func (x *SLOStatus) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[23] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1530,7 +1577,7 @@ func (x *SLOStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use SLOStatus.ProtoReflect.Descriptor instead. func (*SLOStatus) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{23} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{24} } func (x *SLOStatus) GetState() SLOStatusState { @@ -1551,7 +1598,7 @@ type SLOPreviewResponse struct { func (x *SLOPreviewResponse) Reset() { *x = SLOPreviewResponse{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[24] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1564,7 +1611,7 @@ func (x *SLOPreviewResponse) String() string { func (*SLOPreviewResponse) ProtoMessage() {} func (x *SLOPreviewResponse) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[24] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1577,7 +1624,7 @@ func (x *SLOPreviewResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SLOPreviewResponse.ProtoReflect.Descriptor instead. func (*SLOPreviewResponse) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{24} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{25} } func (x *SLOPreviewResponse) GetPlotVector() *PlotVector { @@ -1600,7 +1647,7 @@ type PlotVector struct { func (x *PlotVector) Reset() { *x = PlotVector{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[25] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1613,7 +1660,7 @@ func (x *PlotVector) String() string { func (*PlotVector) ProtoMessage() {} func (x *PlotVector) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[25] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1626,7 +1673,7 @@ func (x *PlotVector) ProtoReflect() protoreflect.Message { // Deprecated: Use PlotVector.ProtoReflect.Descriptor instead. func (*PlotVector) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{25} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{26} } func (x *PlotVector) GetObjective() float64 { @@ -1662,7 +1709,7 @@ type DataPoint struct { func (x *DataPoint) Reset() { *x = DataPoint{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[26] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1675,7 +1722,7 @@ func (x *DataPoint) String() string { func (*DataPoint) ProtoMessage() {} func (x *DataPoint) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[26] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1688,7 +1735,7 @@ func (x *DataPoint) ProtoReflect() protoreflect.Message { // Deprecated: Use DataPoint.ProtoReflect.Descriptor instead. func (*DataPoint) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{26} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{27} } func (x *DataPoint) GetSli() float64 { @@ -1718,7 +1765,7 @@ type AlertFiringWindows struct { func (x *AlertFiringWindows) Reset() { *x = AlertFiringWindows{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[27] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1731,7 +1778,7 @@ func (x *AlertFiringWindows) String() string { func (*AlertFiringWindows) ProtoMessage() {} func (x *AlertFiringWindows) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[27] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1744,7 +1791,7 @@ func (x *AlertFiringWindows) ProtoReflect() protoreflect.Message { // Deprecated: Use AlertFiringWindows.ProtoReflect.Descriptor instead. func (*AlertFiringWindows) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{27} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{28} } func (x *AlertFiringWindows) GetStart() *timestamppb.Timestamp { @@ -1779,7 +1826,7 @@ type ListServiceRequest struct { func (x *ListServiceRequest) Reset() { *x = ListServiceRequest{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[28] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1792,7 +1839,7 @@ func (x *ListServiceRequest) String() string { func (*ListServiceRequest) ProtoMessage() {} func (x *ListServiceRequest) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[28] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[29] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1805,7 +1852,7 @@ func (x *ListServiceRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListServiceRequest.ProtoReflect.Descriptor instead. func (*ListServiceRequest) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{28} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{29} } func (x *ListServiceRequest) GetDatasource() string { @@ -1815,431 +1862,454 @@ func (x *ListServiceRequest) GetDatasource() string { return "" } -var File_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto protoreflect.FileDescriptor - -var file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDesc = []byte{ - 0x0a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, - 0x63, 0x68, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, - 0x73, 0x2f, 0x73, 0x6c, 0x6f, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x73, 0x6c, 0x6f, 0x2f, 0x73, - 0x6c, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x03, 0x73, 0x6c, 0x6f, 0x1a, 0x1e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, - 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, - 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x15, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x68, 0x74, 0x74, 0x70, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x1a, 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x61, - 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x1a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, +var File_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto protoreflect.FileDescriptor + +var file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDesc = []byte{ + 0x0a, 0x31, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, - 0x69, 0x73, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x17, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x72, 0x70, - 0x63, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x3b, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, - 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, - 0x2f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x76, 0x31, 0x2f, 0x61, 0x6c, 0x65, - 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x45, 0x67, 0x69, 0x74, + 0x69, 0x73, 0x2f, 0x73, 0x6c, 0x6f, 0x2f, 0x76, 0x31, 0x2f, 0x73, 0x6c, 0x6f, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x12, 0x03, 0x73, 0x6c, 0x6f, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x15, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x61, + 0x70, 0x69, 0x2f, 0x68, 0x74, 0x74, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1c, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x72, 0x2f, - 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x61, 0x6c, - 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x76, 0x31, 0x2f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, - 0x6e, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x1a, 0x44, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, - 0x61, 0x6e, 0x63, 0x68, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, - 0x61, 0x70, 0x69, 0x73, 0x2f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x76, 0x31, - 0x2f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, - 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x69, 0x0a, 0x0f, 0x4d, 0x75, 0x6c, 0x74, - 0x69, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x4c, 0x4f, 0x12, 0x29, 0x0a, 0x07, 0x63, - 0x6c, 0x6f, 0x6e, 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x07, 0x63, - 0x6c, 0x6f, 0x6e, 0x65, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x08, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, - 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x08, 0x63, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x73, 0x22, 0x32, 0x0a, 0x14, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x43, 0x6c, 0x75, 0x73, - 0x74, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x66, - 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x66, - 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x22, 0x8b, 0x01, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, - 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x1c, 0x0a, - 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x63, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x49, 0x64, 0x22, 0x70, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x64, - 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x22, 0x53, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, + 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x63, 0x6f, + 0x72, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x1a, 0x17, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x72, 0x70, 0x63, 0x2f, 0x73, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x72, 0x2f, 0x6f, 0x70, + 0x6e, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x61, 0x6c, 0x65, 0x72, + 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x76, 0x31, 0x2f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x45, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, + 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, + 0x67, 0x2f, 0x76, 0x31, 0x2f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x63, 0x6f, + 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x44, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, + 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, + 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x76, 0x31, 0x2f, 0x61, 0x6c, 0x65, 0x72, + 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x22, 0x26, 0x0a, 0x0e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x69, 0x0a, 0x0f, 0x4d, + 0x75, 0x6c, 0x74, 0x69, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x4c, 0x4f, 0x12, 0x29, + 0x0a, 0x07, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x0f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x52, 0x07, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x08, 0x63, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x08, 0x63, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x73, 0x22, 0x32, 0x0a, 0x14, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x43, + 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x12, 0x1a, + 0x0a, 0x08, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x08, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x22, 0x8b, 0x01, 0x0a, 0x11, 0x4c, + 0x69, 0x73, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x12, 0x1c, + 0x0a, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, + 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x64, 0x22, 0x70, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, + 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x22, 0x2d, 0x0a, 0x09, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x20, 0x0a, 0x05, 0x69, 0x74, 0x65, - 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x2d, 0x0a, 0x05, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x76, 0x61, 0x6c, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x76, 0x61, 0x6c, 0x73, 0x22, 0xab, 0x01, 0x0a, 0x0b, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x6a, 0x6f, - 0x62, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f, 0x62, 0x49, 0x64, - 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1e, - 0x0a, 0x0a, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x22, - 0x0a, 0x0c, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x64, 0x47, 0x6f, 0x6f, 0x64, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x64, 0x47, 0x6f, - 0x6f, 0x64, 0x12, 0x24, 0x0a, 0x0d, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x64, 0x54, 0x6f, - 0x74, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x49, 0x64, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x22, 0x45, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, - 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x02, + 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x22, 0x53, 0x0a, 0x13, 0x4c, 0x69, + 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x22, - 0x31, 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x22, - 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, - 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x05, 0x69, 0x74, 0x65, - 0x6d, 0x73, 0x22, 0x1b, 0x0a, 0x05, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, - 0xe7, 0x02, 0x0a, 0x05, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2e, 0x0a, - 0x12, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x6e, 0x6f, 0x74, 0x69, 0x66, - 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x38, 0x0a, - 0x17, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x17, - 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x73, 0x63, - 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, - 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, - 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, - 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x24, 0x0a, 0x0d, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x54, 0x79, 0x70, 0x65, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, - 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6f, 0x6e, 0x4e, 0x6f, 0x44, 0x61, 0x74, - 0x61, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x6f, 0x6e, 0x4e, 0x6f, 0x44, 0x61, 0x74, - 0x61, 0x12, 0x1a, 0x0a, 0x08, 0x6f, 0x6e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x18, 0x08, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x08, 0x6f, 0x6e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x1a, 0x0a, - 0x08, 0x6f, 0x6e, 0x42, 0x72, 0x65, 0x61, 0x63, 0x68, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x08, 0x6f, 0x6e, 0x42, 0x72, 0x65, 0x61, 0x63, 0x68, 0x12, 0x1e, 0x0a, 0x0a, 0x6f, 0x6e, 0x52, - 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x6f, - 0x6e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x22, 0xae, 0x04, 0x0a, 0x15, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x69, 0x76, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, - 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, - 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x49, 0x64, 0x12, 0x26, 0x0a, 0x0e, 0x67, 0x6f, 0x6f, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x67, 0x6f, 0x6f, - 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x74, - 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x0a, 0x67, 0x6f, 0x6f, 0x64, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x0a, 0x67, 0x6f, 0x6f, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, - 0x73, 0x12, 0x2c, 0x0a, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, - 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x52, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, - 0x1c, 0x0a, 0x09, 0x73, 0x6c, 0x6f, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x18, 0x09, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x73, 0x6c, 0x6f, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x12, 0x47, 0x0a, - 0x11, 0x62, 0x75, 0x64, 0x67, 0x65, 0x74, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, - 0x61, 0x6c, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x11, 0x62, 0x75, 0x64, 0x67, 0x65, 0x74, 0x69, 0x6e, 0x67, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x23, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x54, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x22, 0x0a, 0x06, 0x6c, - 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x73, 0x6c, - 0x6f, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, - 0x49, 0x0a, 0x11, 0x61, 0x74, 0x74, 0x61, 0x63, 0x68, 0x65, 0x64, 0x45, 0x6e, 0x64, 0x70, 0x6f, - 0x69, 0x6e, 0x74, 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x61, 0x6c, 0x65, - 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x65, 0x64, 0x45, 0x6e, - 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x52, 0x11, 0x61, 0x74, 0x74, 0x61, 0x63, 0x68, 0x65, - 0x64, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x22, 0x40, 0x0a, 0x10, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x53, 0x4c, 0x4f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2c, - 0x0a, 0x03, 0x73, 0x6c, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73, 0x6c, - 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x4f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x52, 0x03, 0x73, 0x6c, 0x6f, 0x22, 0x35, 0x0a, 0x0d, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x50, 0x61, 0x69, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x24, 0x0a, - 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x73, - 0x6c, 0x6f, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x50, 0x61, 0x69, 0x72, 0x52, 0x05, 0x69, 0x74, - 0x65, 0x6d, 0x73, 0x22, 0x31, 0x0a, 0x09, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x50, 0x61, 0x69, 0x72, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x76, 0x61, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x04, 0x76, 0x61, 0x6c, 0x73, 0x22, 0x81, 0x01, 0x0a, 0x07, 0x53, 0x4c, 0x4f, 0x44, 0x61, - 0x74, 0x61, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, - 0x69, 0x64, 0x12, 0x2c, 0x0a, 0x03, 0x53, 0x4c, 0x4f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x65, 0x76, - 0x65, 0x6c, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x52, 0x03, 0x53, 0x4c, 0x4f, - 0x12, 0x38, 0x0a, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, - 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x22, 0x1e, 0x0a, 0x06, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x49, 0x0a, 0x06, 0x4d, 0x65, - 0x74, 0x72, 0x69, 0x63, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x02, 0x69, 0x64, 0x12, 0x2f, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0x5a, 0x0a, 0x0e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, - 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, - 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, - 0x04, 0x75, 0x6e, 0x69, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x6e, 0x69, - 0x74, 0x22, 0x2f, 0x0a, 0x0a, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4c, 0x69, 0x73, 0x74, 0x12, - 0x21, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, - 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x05, 0x69, 0x74, 0x65, - 0x6d, 0x73, 0x22, 0xc7, 0x01, 0x0a, 0x0f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x47, 0x72, 0x6f, - 0x75, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x5c, 0x0a, 0x12, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x4e, - 0x61, 0x6d, 0x65, 0x54, 0x6f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x47, - 0x72, 0x6f, 0x75, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x2e, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x61, - 0x6d, 0x65, 0x54, 0x6f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x12, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x54, 0x6f, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x73, 0x1a, 0x56, 0x0a, 0x17, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x61, 0x6d, - 0x65, 0x54, 0x6f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x25, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x0f, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4c, 0x69, 0x73, - 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x3f, 0x0a, 0x19, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x69, 0x76, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x22, 0x0a, 0x05, 0x69, 0x74, 0x65, - 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, - 0x4c, 0x4f, 0x44, 0x61, 0x74, 0x61, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x36, 0x0a, - 0x09, 0x53, 0x4c, 0x4f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x29, 0x0a, 0x05, 0x73, 0x74, - 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x13, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, - 0x53, 0x4c, 0x4f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x45, 0x0a, 0x12, 0x53, 0x4c, 0x4f, 0x50, 0x72, 0x65, 0x76, - 0x69, 0x65, 0x77, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x0a, 0x70, - 0x6c, 0x6f, 0x74, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x0f, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x50, 0x6c, 0x6f, 0x74, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x52, 0x0a, 0x70, 0x6c, 0x6f, 0x74, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x83, 0x01, 0x0a, - 0x0a, 0x50, 0x6c, 0x6f, 0x74, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x1c, 0x0a, 0x09, 0x6f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, - 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x12, 0x24, 0x0a, 0x05, 0x69, 0x74, 0x65, - 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x44, - 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x12, - 0x31, 0x0a, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x17, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x46, 0x69, 0x72, 0x69, - 0x6e, 0x67, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x52, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f, - 0x77, 0x73, 0x22, 0x57, 0x0a, 0x09, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, - 0x10, 0x0a, 0x03, 0x73, 0x6c, 0x69, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x03, 0x73, 0x6c, - 0x69, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, + 0x2d, 0x0a, 0x09, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x20, 0x0a, 0x05, + 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x73, 0x6c, + 0x6f, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x2d, + 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x76, 0x61, 0x6c, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x76, 0x61, 0x6c, 0x73, 0x22, 0xab, 0x01, + 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x14, 0x0a, + 0x05, 0x6a, 0x6f, 0x62, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f, + 0x62, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, + 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x64, 0x47, 0x6f, 0x6f, + 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, + 0x64, 0x47, 0x6f, 0x6f, 0x64, 0x12, 0x24, 0x0a, 0x0d, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, + 0x64, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x49, 0x64, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x22, 0x45, 0x0a, 0x07, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, + 0x49, 0x64, 0x22, 0x31, 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x69, 0x73, + 0x74, 0x12, 0x22, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x0c, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x05, + 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x1b, 0x0a, 0x05, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x22, 0xe7, 0x02, 0x0a, 0x05, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x2e, 0x0a, 0x12, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x6e, 0x6f, + 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x12, 0x38, 0x0a, 0x17, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x17, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x44, + 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0d, + 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, + 0x70, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x54, + 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x68, 0x72, 0x65, 0x73, + 0x68, 0x6f, 0x6c, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6f, 0x6e, 0x4e, 0x6f, + 0x44, 0x61, 0x74, 0x61, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x6f, 0x6e, 0x4e, 0x6f, + 0x44, 0x61, 0x74, 0x61, 0x12, 0x1a, 0x0a, 0x08, 0x6f, 0x6e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x6f, 0x6e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x12, 0x1a, 0x0a, 0x08, 0x6f, 0x6e, 0x42, 0x72, 0x65, 0x61, 0x63, 0x68, 0x18, 0x09, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x08, 0x6f, 0x6e, 0x42, 0x72, 0x65, 0x61, 0x63, 0x68, 0x12, 0x1e, 0x0a, 0x0a, + 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x0a, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x22, 0xae, 0x04, 0x0a, + 0x15, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x4f, 0x62, 0x6a, + 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x61, + 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, + 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x12, 0x26, 0x0a, 0x0e, 0x67, 0x6f, 0x6f, 0x64, 0x4d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, + 0x67, 0x6f, 0x6f, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x28, + 0x0a, 0x0f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, + 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x0a, 0x67, 0x6f, 0x6f, 0x64, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x73, + 0x6c, 0x6f, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x0a, 0x67, 0x6f, 0x6f, 0x64, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x73, 0x12, 0x2c, 0x0a, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x6c, 0x6f, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x18, + 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x6c, 0x6f, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, + 0x12, 0x47, 0x0a, 0x11, 0x62, 0x75, 0x64, 0x67, 0x65, 0x74, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x74, + 0x65, 0x72, 0x76, 0x61, 0x6c, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x11, 0x62, 0x75, 0x64, 0x67, 0x65, 0x74, 0x69, 0x6e, + 0x67, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x23, 0x0a, 0x06, 0x74, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, + 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x22, + 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, + 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, + 0x6c, 0x73, 0x12, 0x49, 0x0a, 0x11, 0x61, 0x74, 0x74, 0x61, 0x63, 0x68, 0x65, 0x64, 0x45, 0x6e, + 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, + 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x65, + 0x64, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x52, 0x11, 0x61, 0x74, 0x74, 0x61, + 0x63, 0x68, 0x65, 0x64, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x22, 0x40, 0x0a, + 0x10, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x4c, 0x4f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x73, 0x6c, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x65, 0x76, 0x65, + 0x6c, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x52, 0x03, 0x73, 0x6c, 0x6f, 0x22, + 0x35, 0x0a, 0x0d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x50, 0x61, 0x69, 0x72, 0x4c, 0x69, 0x73, 0x74, + 0x12, 0x24, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x0e, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x50, 0x61, 0x69, 0x72, 0x52, + 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x31, 0x0a, 0x09, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x50, + 0x61, 0x69, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x76, 0x61, 0x6c, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x04, 0x76, 0x61, 0x6c, 0x73, 0x22, 0x81, 0x01, 0x0a, 0x07, 0x53, 0x4c, + 0x4f, 0x44, 0x61, 0x74, 0x61, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2c, 0x0a, 0x03, 0x53, 0x4c, 0x4f, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x52, 0x03, + 0x53, 0x4c, 0x4f, 0x12, 0x38, 0x0a, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x22, 0x1e, 0x0a, + 0x06, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x49, 0x0a, + 0x06, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2f, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, + 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0x5a, 0x0a, 0x0e, 0x4d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, + 0x12, 0x12, 0x0a, 0x04, 0x75, 0x6e, 0x69, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x75, 0x6e, 0x69, 0x74, 0x22, 0x2f, 0x0a, 0x0a, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4c, 0x69, + 0x73, 0x74, 0x12, 0x21, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x0b, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x05, + 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0xc7, 0x01, 0x0a, 0x0f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x5c, 0x0a, 0x12, 0x67, 0x72, 0x6f, + 0x75, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x54, 0x6f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x2e, 0x47, 0x72, 0x6f, 0x75, + 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x54, 0x6f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x12, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x54, 0x6f, + 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x1a, 0x56, 0x0a, 0x17, 0x47, 0x72, 0x6f, 0x75, 0x70, + 0x4e, 0x61, 0x6d, 0x65, 0x54, 0x6f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x25, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, + 0x3f, 0x0a, 0x19, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x4f, + 0x62, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x22, 0x0a, 0x05, + 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x73, 0x6c, + 0x6f, 0x2e, 0x53, 0x4c, 0x4f, 0x44, 0x61, 0x74, 0x61, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, + 0x22, 0x36, 0x0a, 0x09, 0x53, 0x4c, 0x4f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x29, 0x0a, + 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x13, 0x2e, 0x73, + 0x6c, 0x6f, 0x2e, 0x53, 0x4c, 0x4f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x45, 0x0a, 0x12, 0x53, 0x4c, 0x4f, 0x50, + 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, + 0x0a, 0x0a, 0x70, 0x6c, 0x6f, 0x74, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x50, 0x6c, 0x6f, 0x74, 0x56, 0x65, 0x63, + 0x74, 0x6f, 0x72, 0x52, 0x0a, 0x70, 0x6c, 0x6f, 0x74, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, + 0x83, 0x01, 0x0a, 0x0a, 0x50, 0x6c, 0x6f, 0x74, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x1c, + 0x0a, 0x09, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x01, 0x52, 0x09, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x12, 0x24, 0x0a, 0x05, + 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x73, 0x6c, + 0x6f, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x05, 0x69, 0x74, 0x65, + 0x6d, 0x73, 0x12, 0x31, 0x0a, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x18, 0x03, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x46, + 0x69, 0x72, 0x69, 0x6e, 0x67, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x52, 0x07, 0x77, 0x69, + 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x22, 0x57, 0x0a, 0x09, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, + 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x6c, 0x69, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, + 0x03, 0x73, 0x6c, 0x69, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x90, + 0x01, 0x0a, 0x12, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x46, 0x69, 0x72, 0x69, 0x6e, 0x67, 0x57, 0x69, + 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x90, 0x01, 0x0a, 0x12, - 0x41, 0x6c, 0x65, 0x72, 0x74, 0x46, 0x69, 0x72, 0x69, 0x6e, 0x67, 0x57, 0x69, 0x6e, 0x64, 0x6f, - 0x77, 0x73, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, - 0x74, 0x61, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x65, - 0x6e, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x22, 0x34, - 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x2a, 0x84, 0x01, 0x0a, 0x0e, 0x53, 0x4c, 0x4f, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x49, 0x6e, 0x50, 0x72, 0x6f, - 0x67, 0x72, 0x65, 0x73, 0x73, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x72, 0x65, 0x61, 0x74, - 0x69, 0x6e, 0x67, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x4e, 0x6f, 0x44, 0x61, 0x74, 0x61, 0x10, - 0x02, 0x12, 0x06, 0x0a, 0x02, 0x4f, 0x6b, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x57, 0x61, 0x72, - 0x6e, 0x69, 0x6e, 0x67, 0x10, 0x04, 0x12, 0x0d, 0x0a, 0x09, 0x42, 0x72, 0x65, 0x61, 0x63, 0x68, - 0x69, 0x6e, 0x67, 0x10, 0x05, 0x12, 0x11, 0x0a, 0x0d, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, - 0x44, 0x61, 0x74, 0x61, 0x4f, 0x6b, 0x10, 0x06, 0x12, 0x11, 0x0a, 0x0d, 0x49, 0x6e, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x10, 0x07, 0x32, 0xb6, 0x07, 0x0a, 0x03, - 0x53, 0x4c, 0x4f, 0x12, 0x45, 0x0a, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x4c, 0x4f, - 0x12, 0x15, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x4c, 0x4f, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, - 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x22, 0x10, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0a, - 0x3a, 0x01, 0x2a, 0x22, 0x05, 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x12, 0x3b, 0x0a, 0x06, 0x47, 0x65, - 0x74, 0x53, 0x4c, 0x4f, 0x12, 0x0f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, + 0x52, 0x03, 0x65, 0x6e, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, + 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, + 0x79, 0x22, 0x34, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, + 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2a, 0x84, 0x01, 0x0a, 0x0e, 0x53, 0x4c, 0x4f, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x49, 0x6e, + 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x72, + 0x65, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x4e, 0x6f, 0x44, 0x61, + 0x74, 0x61, 0x10, 0x02, 0x12, 0x06, 0x0a, 0x02, 0x4f, 0x6b, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, + 0x57, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x10, 0x04, 0x12, 0x0d, 0x0a, 0x09, 0x42, 0x72, 0x65, + 0x61, 0x63, 0x68, 0x69, 0x6e, 0x67, 0x10, 0x05, 0x12, 0x11, 0x0a, 0x0d, 0x50, 0x61, 0x72, 0x74, + 0x69, 0x61, 0x6c, 0x44, 0x61, 0x74, 0x61, 0x4f, 0x6b, 0x10, 0x06, 0x12, 0x11, 0x0a, 0x0d, 0x49, + 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x10, 0x07, 0x32, 0x92, + 0x08, 0x0a, 0x03, 0x53, 0x4c, 0x4f, 0x12, 0x45, 0x0a, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x53, 0x4c, 0x4f, 0x12, 0x15, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x53, 0x4c, 0x4f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0f, 0x2e, 0x63, 0x6f, 0x72, + 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x22, 0x10, 0x82, 0xd3, 0xe4, + 0x93, 0x02, 0x0a, 0x3a, 0x01, 0x2a, 0x22, 0x05, 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x12, 0x3b, 0x0a, + 0x06, 0x47, 0x65, 0x74, 0x53, 0x4c, 0x4f, 0x12, 0x0f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, + 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x1a, 0x0c, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, + 0x4c, 0x4f, 0x44, 0x61, 0x74, 0x61, 0x22, 0x12, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0c, 0x22, 0x0a, + 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x2f, 0x7b, 0x69, 0x64, 0x7d, 0x12, 0x51, 0x0a, 0x08, 0x4c, 0x69, + 0x73, 0x74, 0x53, 0x4c, 0x4f, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1e, + 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x65, 0x76, 0x65, + 0x6c, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x0d, + 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x07, 0x12, 0x05, 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x12, 0x4f, 0x0a, + 0x09, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x4c, 0x4f, 0x12, 0x0c, 0x2e, 0x73, 0x6c, 0x6f, + 0x2e, 0x53, 0x4c, 0x4f, 0x44, 0x61, 0x74, 0x61, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, + 0x22, 0x1c, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x16, 0x3a, 0x01, 0x2a, 0x1a, 0x11, 0x2f, 0x73, 0x6c, + 0x6f, 0x73, 0x2f, 0x7b, 0x69, 0x64, 0x7d, 0x2f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x48, + 0x0a, 0x09, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x4c, 0x4f, 0x12, 0x0f, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x1a, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x22, 0x12, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0c, 0x2a, 0x0a, 0x2f, 0x73, + 0x6c, 0x6f, 0x73, 0x2f, 0x7b, 0x69, 0x64, 0x7d, 0x12, 0x43, 0x0a, 0x08, 0x43, 0x6c, 0x6f, 0x6e, + 0x65, 0x53, 0x4c, 0x4f, 0x12, 0x0f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x1a, 0x0c, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x4c, 0x4f, 0x44, - 0x61, 0x74, 0x61, 0x22, 0x12, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0c, 0x22, 0x0a, 0x2f, 0x73, 0x6c, - 0x6f, 0x73, 0x2f, 0x7b, 0x69, 0x64, 0x7d, 0x12, 0x51, 0x0a, 0x08, 0x4c, 0x69, 0x73, 0x74, 0x53, - 0x4c, 0x4f, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1e, 0x2e, 0x73, 0x6c, - 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x4f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x0d, 0x82, 0xd3, 0xe4, - 0x93, 0x02, 0x07, 0x12, 0x05, 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x12, 0x4f, 0x0a, 0x09, 0x55, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x53, 0x4c, 0x4f, 0x12, 0x0c, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x4c, - 0x4f, 0x44, 0x61, 0x74, 0x61, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x1c, 0x82, - 0xd3, 0xe4, 0x93, 0x02, 0x16, 0x3a, 0x01, 0x2a, 0x1a, 0x11, 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x2f, - 0x7b, 0x69, 0x64, 0x7d, 0x2f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x48, 0x0a, 0x09, 0x44, - 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x4c, 0x4f, 0x12, 0x0f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, - 0x79, 0x22, 0x12, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0c, 0x2a, 0x0a, 0x2f, 0x73, 0x6c, 0x6f, 0x73, - 0x2f, 0x7b, 0x69, 0x64, 0x7d, 0x12, 0x43, 0x0a, 0x08, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x53, 0x4c, - 0x4f, 0x12, 0x0f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, - 0x63, 0x65, 0x1a, 0x0c, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x4c, 0x4f, 0x44, 0x61, 0x74, 0x61, - 0x22, 0x18, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x12, 0x22, 0x10, 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x2f, - 0x7b, 0x69, 0x64, 0x7d, 0x2f, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x12, 0x71, 0x0a, 0x0f, 0x43, 0x6c, - 0x6f, 0x6e, 0x65, 0x54, 0x6f, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x73, 0x12, 0x14, 0x2e, - 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x53, 0x4c, 0x4f, 0x1a, 0x19, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x43, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x22, 0x2d, - 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x27, 0x3a, 0x01, 0x2a, 0x22, 0x22, 0x2f, 0x73, 0x6c, 0x6f, 0x73, - 0x2f, 0x7b, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x49, 0x64, 0x2e, 0x69, 0x64, 0x7d, 0x2f, 0x63, 0x6c, - 0x6f, 0x6e, 0x65, 0x54, 0x6f, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x73, 0x12, 0x51, 0x0a, - 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x17, 0x2e, 0x73, - 0x6c, 0x6f, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x72, - 0x69, 0x63, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x13, 0x82, 0xd3, 0xe4, - 0x93, 0x02, 0x0d, 0x3a, 0x01, 0x2a, 0x22, 0x08, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, - 0x12, 0x50, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x12, 0x18, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x73, 0x6c, 0x6f, - 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x14, 0x82, 0xd3, - 0xe4, 0x93, 0x02, 0x0e, 0x3a, 0x01, 0x2a, 0x22, 0x09, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x73, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, - 0x12, 0x16, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x0f, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x09, - 0x22, 0x07, 0x2f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x44, 0x0a, 0x06, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x0f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, - 0x65, 0x6e, 0x63, 0x65, 0x1a, 0x0e, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x4c, 0x4f, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x22, 0x19, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x13, 0x22, 0x11, 0x2f, 0x73, - 0x6c, 0x6f, 0x73, 0x2f, 0x7b, 0x69, 0x64, 0x7d, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, - 0x53, 0x0a, 0x07, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x12, 0x15, 0x2e, 0x73, 0x6c, 0x6f, - 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x4c, 0x4f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x17, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x4c, 0x4f, 0x50, 0x72, 0x65, 0x76, 0x69, - 0x65, 0x77, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x18, 0x82, 0xd3, 0xe4, 0x93, - 0x02, 0x12, 0x3a, 0x01, 0x2a, 0x22, 0x0d, 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x2f, 0x70, 0x72, 0x65, - 0x76, 0x69, 0x65, 0x77, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, - 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x73, 0x2f, 0x73, 0x6c, 0x6f, 0x2f, 0x61, 0x70, 0x69, 0x73, - 0x2f, 0x73, 0x6c, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x74, 0x61, 0x22, 0x18, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x12, 0x22, 0x10, 0x2f, 0x73, 0x6c, + 0x6f, 0x73, 0x2f, 0x7b, 0x69, 0x64, 0x7d, 0x2f, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x12, 0x71, 0x0a, + 0x0f, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x54, 0x6f, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x73, + 0x12, 0x14, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x43, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x53, 0x4c, 0x4f, 0x1a, 0x19, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x75, 0x6c, + 0x74, 0x69, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, + 0x73, 0x22, 0x2d, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x27, 0x3a, 0x01, 0x2a, 0x22, 0x22, 0x2f, 0x73, + 0x6c, 0x6f, 0x73, 0x2f, 0x7b, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x49, 0x64, 0x2e, 0x69, 0x64, 0x7d, + 0x2f, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x54, 0x6f, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x73, + 0x12, 0x51, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, + 0x17, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x13, + 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0d, 0x3a, 0x01, 0x2a, 0x22, 0x08, 0x2f, 0x6d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x73, 0x12, 0x50, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x12, 0x18, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, + 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x22, + 0x14, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0e, 0x3a, 0x01, 0x2a, 0x22, 0x09, 0x2f, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x73, 0x12, 0x16, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x73, 0x6c, + 0x6f, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x0f, 0x82, 0xd3, 0xe4, + 0x93, 0x02, 0x09, 0x22, 0x07, 0x2f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x4a, 0x0a, 0x0c, + 0x47, 0x65, 0x74, 0x53, 0x4c, 0x4f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0f, 0x2e, 0x63, + 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x1a, 0x0e, 0x2e, + 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x4c, 0x4f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x19, 0x82, + 0xd3, 0xe4, 0x93, 0x02, 0x13, 0x22, 0x11, 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x2f, 0x7b, 0x69, 0x64, + 0x7d, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x53, 0x0a, 0x07, 0x50, 0x72, 0x65, 0x76, + 0x69, 0x65, 0x77, 0x12, 0x15, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x53, 0x4c, 0x4f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x73, 0x6c, 0x6f, + 0x2e, 0x53, 0x4c, 0x4f, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x18, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x12, 0x3a, 0x01, 0x2a, 0x22, 0x0d, + 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x2f, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x12, 0x54, 0x0a, + 0x0f, 0x4c, 0x69, 0x73, 0x74, 0x44, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, + 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x13, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x44, + 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0x14, 0x82, + 0xd3, 0xe4, 0x93, 0x02, 0x0e, 0x12, 0x0c, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x73, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, + 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x73, 0x6c, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescOnce sync.Once - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescData = file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDesc + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescOnce sync.Once + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescData = file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDesc ) -func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP() []byte { - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescOnce.Do(func() { - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescData = protoimpl.X.CompressGZIP(file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescData) +func file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP() []byte { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescOnce.Do(func() { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescData = protoimpl.X.CompressGZIP(file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescData) }) - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescData + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescData } -var file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes = make([]protoimpl.MessageInfo, 30) -var file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_goTypes = []interface{}{ +var file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes = make([]protoimpl.MessageInfo, 31) +var file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_goTypes = []interface{}{ (SLOStatusState)(0), // 0: slo.SLOStatusState - (*MultiClusterSLO)(nil), // 1: slo.MultiClusterSLO - (*MultiClusterFailures)(nil), // 2: slo.MultiClusterFailures - (*ListEventsRequest)(nil), // 3: slo.ListEventsRequest - (*ListMetricsRequest)(nil), // 4: slo.ListMetricsRequest - (*ListServicesRequest)(nil), // 5: slo.ListServicesRequest - (*EventList)(nil), // 6: slo.EventList - (*Event)(nil), // 7: slo.Event - (*ServiceInfo)(nil), // 8: slo.ServiceInfo - (*Service)(nil), // 9: slo.Service - (*ServiceList)(nil), // 10: slo.ServiceList - (*Label)(nil), // 11: slo.Label - (*Alert)(nil), // 12: slo.Alert - (*ServiceLevelObjective)(nil), // 13: slo.ServiceLevelObjective - (*CreateSLORequest)(nil), // 14: slo.CreateSLORequest - (*EventPairList)(nil), // 15: slo.EventPairList - (*EventPair)(nil), // 16: slo.EventPair - (*SLOData)(nil), // 17: slo.SLOData - (*Target)(nil), // 18: slo.Target - (*Metric)(nil), // 19: slo.Metric - (*MetricMetadata)(nil), // 20: slo.MetricMetadata - (*MetricList)(nil), // 21: slo.MetricList - (*MetricGroupList)(nil), // 22: slo.MetricGroupList - (*ServiceLevelObjectiveList)(nil), // 23: slo.ServiceLevelObjectiveList - (*SLOStatus)(nil), // 24: slo.SLOStatus - (*SLOPreviewResponse)(nil), // 25: slo.SLOPreviewResponse - (*PlotVector)(nil), // 26: slo.PlotVector - (*DataPoint)(nil), // 27: slo.DataPoint - (*AlertFiringWindows)(nil), // 28: slo.AlertFiringWindows - (*ListServiceRequest)(nil), // 29: slo.ListServiceRequest - nil, // 30: slo.MetricGroupList.GroupNameToMetricsEntry - (*v1.Reference)(nil), // 31: core.Reference - (*durationpb.Duration)(nil), // 32: google.protobuf.Duration - (*v11.AttachedEndpoints)(nil), // 33: alerting.AttachedEndpoints - (*timestamppb.Timestamp)(nil), // 34: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 35: google.protobuf.Empty -} -var file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_depIdxs = []int32{ - 31, // 0: slo.MultiClusterSLO.cloneId:type_name -> core.Reference - 31, // 1: slo.MultiClusterSLO.clusters:type_name -> core.Reference - 7, // 2: slo.EventList.items:type_name -> slo.Event - 9, // 3: slo.ServiceList.items:type_name -> slo.Service - 7, // 4: slo.ServiceLevelObjective.goodEvents:type_name -> slo.Event - 7, // 5: slo.ServiceLevelObjective.totalEvents:type_name -> slo.Event - 32, // 6: slo.ServiceLevelObjective.budgetingInterval:type_name -> google.protobuf.Duration - 18, // 7: slo.ServiceLevelObjective.target:type_name -> slo.Target - 11, // 8: slo.ServiceLevelObjective.labels:type_name -> slo.Label - 33, // 9: slo.ServiceLevelObjective.attachedEndpoints:type_name -> alerting.AttachedEndpoints - 13, // 10: slo.CreateSLORequest.slo:type_name -> slo.ServiceLevelObjective - 16, // 11: slo.EventPairList.items:type_name -> slo.EventPair - 13, // 12: slo.SLOData.SLO:type_name -> slo.ServiceLevelObjective - 34, // 13: slo.SLOData.createdAt:type_name -> google.protobuf.Timestamp - 20, // 14: slo.Metric.metadata:type_name -> slo.MetricMetadata - 19, // 15: slo.MetricList.items:type_name -> slo.Metric - 30, // 16: slo.MetricGroupList.groupNameToMetrics:type_name -> slo.MetricGroupList.GroupNameToMetricsEntry - 17, // 17: slo.ServiceLevelObjectiveList.items:type_name -> slo.SLOData + (*DatasourceInfo)(nil), // 1: slo.DatasourceInfo + (*MultiClusterSLO)(nil), // 2: slo.MultiClusterSLO + (*MultiClusterFailures)(nil), // 3: slo.MultiClusterFailures + (*ListEventsRequest)(nil), // 4: slo.ListEventsRequest + (*ListMetricsRequest)(nil), // 5: slo.ListMetricsRequest + (*ListServicesRequest)(nil), // 6: slo.ListServicesRequest + (*EventList)(nil), // 7: slo.EventList + (*Event)(nil), // 8: slo.Event + (*ServiceInfo)(nil), // 9: slo.ServiceInfo + (*Service)(nil), // 10: slo.Service + (*ServiceList)(nil), // 11: slo.ServiceList + (*Label)(nil), // 12: slo.Label + (*Alert)(nil), // 13: slo.Alert + (*ServiceLevelObjective)(nil), // 14: slo.ServiceLevelObjective + (*CreateSLORequest)(nil), // 15: slo.CreateSLORequest + (*EventPairList)(nil), // 16: slo.EventPairList + (*EventPair)(nil), // 17: slo.EventPair + (*SLOData)(nil), // 18: slo.SLOData + (*Target)(nil), // 19: slo.Target + (*Metric)(nil), // 20: slo.Metric + (*MetricMetadata)(nil), // 21: slo.MetricMetadata + (*MetricList)(nil), // 22: slo.MetricList + (*MetricGroupList)(nil), // 23: slo.MetricGroupList + (*ServiceLevelObjectiveList)(nil), // 24: slo.ServiceLevelObjectiveList + (*SLOStatus)(nil), // 25: slo.SLOStatus + (*SLOPreviewResponse)(nil), // 26: slo.SLOPreviewResponse + (*PlotVector)(nil), // 27: slo.PlotVector + (*DataPoint)(nil), // 28: slo.DataPoint + (*AlertFiringWindows)(nil), // 29: slo.AlertFiringWindows + (*ListServiceRequest)(nil), // 30: slo.ListServiceRequest + nil, // 31: slo.MetricGroupList.GroupNameToMetricsEntry + (*v1.Reference)(nil), // 32: core.Reference + (*durationpb.Duration)(nil), // 33: google.protobuf.Duration + (*v11.AttachedEndpoints)(nil), // 34: alerting.AttachedEndpoints + (*timestamppb.Timestamp)(nil), // 35: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 36: google.protobuf.Empty +} +var file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_depIdxs = []int32{ + 32, // 0: slo.MultiClusterSLO.cloneId:type_name -> core.Reference + 32, // 1: slo.MultiClusterSLO.clusters:type_name -> core.Reference + 8, // 2: slo.EventList.items:type_name -> slo.Event + 10, // 3: slo.ServiceList.items:type_name -> slo.Service + 8, // 4: slo.ServiceLevelObjective.goodEvents:type_name -> slo.Event + 8, // 5: slo.ServiceLevelObjective.totalEvents:type_name -> slo.Event + 33, // 6: slo.ServiceLevelObjective.budgetingInterval:type_name -> google.protobuf.Duration + 19, // 7: slo.ServiceLevelObjective.target:type_name -> slo.Target + 12, // 8: slo.ServiceLevelObjective.labels:type_name -> slo.Label + 34, // 9: slo.ServiceLevelObjective.attachedEndpoints:type_name -> alerting.AttachedEndpoints + 14, // 10: slo.CreateSLORequest.slo:type_name -> slo.ServiceLevelObjective + 17, // 11: slo.EventPairList.items:type_name -> slo.EventPair + 14, // 12: slo.SLOData.SLO:type_name -> slo.ServiceLevelObjective + 35, // 13: slo.SLOData.createdAt:type_name -> google.protobuf.Timestamp + 21, // 14: slo.Metric.metadata:type_name -> slo.MetricMetadata + 20, // 15: slo.MetricList.items:type_name -> slo.Metric + 31, // 16: slo.MetricGroupList.groupNameToMetrics:type_name -> slo.MetricGroupList.GroupNameToMetricsEntry + 18, // 17: slo.ServiceLevelObjectiveList.items:type_name -> slo.SLOData 0, // 18: slo.SLOStatus.state:type_name -> slo.SLOStatusState - 26, // 19: slo.SLOPreviewResponse.plotVector:type_name -> slo.PlotVector - 27, // 20: slo.PlotVector.items:type_name -> slo.DataPoint - 28, // 21: slo.PlotVector.windows:type_name -> slo.AlertFiringWindows - 34, // 22: slo.DataPoint.timestamp:type_name -> google.protobuf.Timestamp - 34, // 23: slo.AlertFiringWindows.start:type_name -> google.protobuf.Timestamp - 34, // 24: slo.AlertFiringWindows.end:type_name -> google.protobuf.Timestamp - 21, // 25: slo.MetricGroupList.GroupNameToMetricsEntry.value:type_name -> slo.MetricList - 14, // 26: slo.SLO.CreateSLO:input_type -> slo.CreateSLORequest - 31, // 27: slo.SLO.GetSLO:input_type -> core.Reference - 35, // 28: slo.SLO.ListSLOs:input_type -> google.protobuf.Empty - 17, // 29: slo.SLO.UpdateSLO:input_type -> slo.SLOData - 31, // 30: slo.SLO.DeleteSLO:input_type -> core.Reference - 31, // 31: slo.SLO.CloneSLO:input_type -> core.Reference - 1, // 32: slo.SLO.CloneToClusters:input_type -> slo.MultiClusterSLO - 4, // 33: slo.SLO.ListMetrics:input_type -> slo.ListMetricsRequest - 5, // 34: slo.SLO.ListServices:input_type -> slo.ListServicesRequest - 3, // 35: slo.SLO.ListEvents:input_type -> slo.ListEventsRequest - 31, // 36: slo.SLO.Status:input_type -> core.Reference - 14, // 37: slo.SLO.Preview:input_type -> slo.CreateSLORequest - 31, // 38: slo.SLO.CreateSLO:output_type -> core.Reference - 17, // 39: slo.SLO.GetSLO:output_type -> slo.SLOData - 23, // 40: slo.SLO.ListSLOs:output_type -> slo.ServiceLevelObjectiveList - 35, // 41: slo.SLO.UpdateSLO:output_type -> google.protobuf.Empty - 35, // 42: slo.SLO.DeleteSLO:output_type -> google.protobuf.Empty - 17, // 43: slo.SLO.CloneSLO:output_type -> slo.SLOData - 2, // 44: slo.SLO.CloneToClusters:output_type -> slo.MultiClusterFailures - 22, // 45: slo.SLO.ListMetrics:output_type -> slo.MetricGroupList - 10, // 46: slo.SLO.ListServices:output_type -> slo.ServiceList - 6, // 47: slo.SLO.ListEvents:output_type -> slo.EventList - 24, // 48: slo.SLO.Status:output_type -> slo.SLOStatus - 25, // 49: slo.SLO.Preview:output_type -> slo.SLOPreviewResponse - 38, // [38:50] is the sub-list for method output_type - 26, // [26:38] is the sub-list for method input_type + 27, // 19: slo.SLOPreviewResponse.plotVector:type_name -> slo.PlotVector + 28, // 20: slo.PlotVector.items:type_name -> slo.DataPoint + 29, // 21: slo.PlotVector.windows:type_name -> slo.AlertFiringWindows + 35, // 22: slo.DataPoint.timestamp:type_name -> google.protobuf.Timestamp + 35, // 23: slo.AlertFiringWindows.start:type_name -> google.protobuf.Timestamp + 35, // 24: slo.AlertFiringWindows.end:type_name -> google.protobuf.Timestamp + 22, // 25: slo.MetricGroupList.GroupNameToMetricsEntry.value:type_name -> slo.MetricList + 15, // 26: slo.SLO.CreateSLO:input_type -> slo.CreateSLORequest + 32, // 27: slo.SLO.GetSLO:input_type -> core.Reference + 36, // 28: slo.SLO.ListSLOs:input_type -> google.protobuf.Empty + 18, // 29: slo.SLO.UpdateSLO:input_type -> slo.SLOData + 32, // 30: slo.SLO.DeleteSLO:input_type -> core.Reference + 32, // 31: slo.SLO.CloneSLO:input_type -> core.Reference + 2, // 32: slo.SLO.CloneToClusters:input_type -> slo.MultiClusterSLO + 5, // 33: slo.SLO.ListMetrics:input_type -> slo.ListMetricsRequest + 6, // 34: slo.SLO.ListServices:input_type -> slo.ListServicesRequest + 4, // 35: slo.SLO.ListEvents:input_type -> slo.ListEventsRequest + 32, // 36: slo.SLO.GetSLOStatus:input_type -> core.Reference + 15, // 37: slo.SLO.Preview:input_type -> slo.CreateSLORequest + 36, // 38: slo.SLO.ListDatasources:input_type -> google.protobuf.Empty + 32, // 39: slo.SLO.CreateSLO:output_type -> core.Reference + 18, // 40: slo.SLO.GetSLO:output_type -> slo.SLOData + 24, // 41: slo.SLO.ListSLOs:output_type -> slo.ServiceLevelObjectiveList + 36, // 42: slo.SLO.UpdateSLO:output_type -> google.protobuf.Empty + 36, // 43: slo.SLO.DeleteSLO:output_type -> google.protobuf.Empty + 18, // 44: slo.SLO.CloneSLO:output_type -> slo.SLOData + 3, // 45: slo.SLO.CloneToClusters:output_type -> slo.MultiClusterFailures + 23, // 46: slo.SLO.ListMetrics:output_type -> slo.MetricGroupList + 11, // 47: slo.SLO.ListServices:output_type -> slo.ServiceList + 7, // 48: slo.SLO.ListEvents:output_type -> slo.EventList + 25, // 49: slo.SLO.GetSLOStatus:output_type -> slo.SLOStatus + 26, // 50: slo.SLO.Preview:output_type -> slo.SLOPreviewResponse + 1, // 51: slo.SLO.ListDatasources:output_type -> slo.DatasourceInfo + 39, // [39:52] is the sub-list for method output_type + 26, // [26:39] is the sub-list for method input_type 26, // [26:26] is the sub-list for extension type_name 26, // [26:26] is the sub-list for extension extendee 0, // [0:26] is the sub-list for field type_name } -func init() { file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() } -func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { - if File_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto != nil { +func init() { file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_init() } +func file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_init() { + if File_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto != nil { return } if !protoimpl.UnsafeEnabled { - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DatasourceInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*MultiClusterSLO); i { case 0: return &v.state @@ -2251,7 +2321,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*MultiClusterFailures); i { case 0: return &v.state @@ -2263,7 +2333,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListEventsRequest); i { case 0: return &v.state @@ -2275,7 +2345,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListMetricsRequest); i { case 0: return &v.state @@ -2287,7 +2357,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListServicesRequest); i { case 0: return &v.state @@ -2299,7 +2369,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*EventList); i { case 0: return &v.state @@ -2311,7 +2381,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Event); i { case 0: return &v.state @@ -2323,7 +2393,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ServiceInfo); i { case 0: return &v.state @@ -2335,7 +2405,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Service); i { case 0: return &v.state @@ -2347,7 +2417,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ServiceList); i { case 0: return &v.state @@ -2359,7 +2429,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Label); i { case 0: return &v.state @@ -2371,7 +2441,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Alert); i { case 0: return &v.state @@ -2383,7 +2453,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ServiceLevelObjective); i { case 0: return &v.state @@ -2395,7 +2465,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*CreateSLORequest); i { case 0: return &v.state @@ -2407,7 +2477,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*EventPairList); i { case 0: return &v.state @@ -2419,7 +2489,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*EventPair); i { case 0: return &v.state @@ -2431,7 +2501,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SLOData); i { case 0: return &v.state @@ -2443,7 +2513,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Target); i { case 0: return &v.state @@ -2455,7 +2525,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Metric); i { case 0: return &v.state @@ -2467,7 +2537,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*MetricMetadata); i { case 0: return &v.state @@ -2479,7 +2549,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*MetricList); i { case 0: return &v.state @@ -2491,7 +2561,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*MetricGroupList); i { case 0: return &v.state @@ -2503,7 +2573,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ServiceLevelObjectiveList); i { case 0: return &v.state @@ -2515,7 +2585,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SLOStatus); i { case 0: return &v.state @@ -2527,7 +2597,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SLOPreviewResponse); i { case 0: return &v.state @@ -2539,7 +2609,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*PlotVector); i { case 0: return &v.state @@ -2551,7 +2621,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DataPoint); i { case 0: return &v.state @@ -2563,7 +2633,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*AlertFiringWindows); i { case 0: return &v.state @@ -2575,7 +2645,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListServiceRequest); i { case 0: return &v.state @@ -2592,19 +2662,19 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDesc, + RawDescriptor: file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDesc, NumEnums: 1, - NumMessages: 30, + NumMessages: 31, NumExtensions: 0, NumServices: 1, }, - GoTypes: file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_goTypes, - DependencyIndexes: file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_depIdxs, - EnumInfos: file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_enumTypes, - MessageInfos: file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes, + GoTypes: file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_goTypes, + DependencyIndexes: file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_depIdxs, + EnumInfos: file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_enumTypes, + MessageInfos: file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes, }.Build() - File_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto = out.File - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDesc = nil - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_goTypes = nil - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_depIdxs = nil + File_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto = out.File + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDesc = nil + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_goTypes = nil + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_depIdxs = nil } diff --git a/plugins/slo/apis/slo/slo.proto b/pkg/apis/slo/v1/slo.proto similarity index 94% rename from plugins/slo/apis/slo/slo.proto rename to pkg/apis/slo/v1/slo.proto index dd6a83934b..051ed1be7b 100644 --- a/plugins/slo/apis/slo/slo.proto +++ b/pkg/apis/slo/v1/slo.proto @@ -1,5 +1,5 @@ syntax = "proto3"; -option go_package = "github.com/rancher/opni/plugins/slo/apis/slo"; +option go_package = "github.com/rancher/opni/pkg/apis/slo/v1"; import "google/protobuf/duration.proto"; import "google/protobuf/empty.proto"; @@ -80,7 +80,7 @@ service SLO { } // Returns a status enum badge for a given SLO - rpc Status(core.Reference) returns (SLOStatus) { + rpc GetSLOStatus(core.Reference) returns (SLOStatus) { option (google.api.http) = { post: "/slos/{id}/status" @@ -93,8 +93,19 @@ service SLO { body: "*" }; } + + rpc ListDatasources(google.protobuf.Empty) returns (DatasourceInfo) { + option (google.api.http) = { + get: "/datasources" + }; + } } +message DatasourceInfo { + repeated string items = 1; +} + + message MultiClusterSLO { core.Reference cloneId = 1; repeated core.Reference clusters = 2; diff --git a/plugins/slo/apis/slo/slo_grpc.pb.go b/pkg/apis/slo/v1/slo_grpc.pb.go similarity index 87% rename from plugins/slo/apis/slo/slo_grpc.pb.go rename to pkg/apis/slo/v1/slo_grpc.pb.go index 51247d3124..cbdf023d72 100644 --- a/plugins/slo/apis/slo/slo_grpc.pb.go +++ b/pkg/apis/slo/v1/slo_grpc.pb.go @@ -2,9 +2,9 @@ // versions: // - protoc-gen-go-grpc v1.3.0 // - ragu v1.0.0 -// source: github.com/rancher/opni/plugins/slo/apis/slo/slo.proto +// source: github.com/rancher/opni/pkg/apis/slo/v1/slo.proto -package slo +package v1 import ( context "context" @@ -31,8 +31,9 @@ const ( SLO_ListMetrics_FullMethodName = "/slo.SLO/ListMetrics" SLO_ListServices_FullMethodName = "/slo.SLO/ListServices" SLO_ListEvents_FullMethodName = "/slo.SLO/ListEvents" - SLO_Status_FullMethodName = "/slo.SLO/Status" + SLO_GetSLOStatus_FullMethodName = "/slo.SLO/GetSLOStatus" SLO_Preview_FullMethodName = "/slo.SLO/Preview" + SLO_ListDatasources_FullMethodName = "/slo.SLO/ListDatasources" ) // SLOClient is the client API for SLO service. @@ -53,8 +54,9 @@ type SLOClient interface { ListServices(ctx context.Context, in *ListServicesRequest, opts ...grpc.CallOption) (*ServiceList, error) ListEvents(ctx context.Context, in *ListEventsRequest, opts ...grpc.CallOption) (*EventList, error) // Returns a status enum badge for a given SLO - Status(ctx context.Context, in *v1.Reference, opts ...grpc.CallOption) (*SLOStatus, error) + GetSLOStatus(ctx context.Context, in *v1.Reference, opts ...grpc.CallOption) (*SLOStatus, error) Preview(ctx context.Context, in *CreateSLORequest, opts ...grpc.CallOption) (*SLOPreviewResponse, error) + ListDatasources(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*DatasourceInfo, error) } type sLOClient struct { @@ -155,9 +157,9 @@ func (c *sLOClient) ListEvents(ctx context.Context, in *ListEventsRequest, opts return out, nil } -func (c *sLOClient) Status(ctx context.Context, in *v1.Reference, opts ...grpc.CallOption) (*SLOStatus, error) { +func (c *sLOClient) GetSLOStatus(ctx context.Context, in *v1.Reference, opts ...grpc.CallOption) (*SLOStatus, error) { out := new(SLOStatus) - err := c.cc.Invoke(ctx, SLO_Status_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, SLO_GetSLOStatus_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -173,6 +175,15 @@ func (c *sLOClient) Preview(ctx context.Context, in *CreateSLORequest, opts ...g return out, nil } +func (c *sLOClient) ListDatasources(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*DatasourceInfo, error) { + out := new(DatasourceInfo) + err := c.cc.Invoke(ctx, SLO_ListDatasources_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // SLOServer is the server API for SLO service. // All implementations must embed UnimplementedSLOServer // for forward compatibility @@ -191,8 +202,9 @@ type SLOServer interface { ListServices(context.Context, *ListServicesRequest) (*ServiceList, error) ListEvents(context.Context, *ListEventsRequest) (*EventList, error) // Returns a status enum badge for a given SLO - Status(context.Context, *v1.Reference) (*SLOStatus, error) + GetSLOStatus(context.Context, *v1.Reference) (*SLOStatus, error) Preview(context.Context, *CreateSLORequest) (*SLOPreviewResponse, error) + ListDatasources(context.Context, *emptypb.Empty) (*DatasourceInfo, error) mustEmbedUnimplementedSLOServer() } @@ -230,12 +242,15 @@ func (UnimplementedSLOServer) ListServices(context.Context, *ListServicesRequest func (UnimplementedSLOServer) ListEvents(context.Context, *ListEventsRequest) (*EventList, error) { return nil, status.Errorf(codes.Unimplemented, "method ListEvents not implemented") } -func (UnimplementedSLOServer) Status(context.Context, *v1.Reference) (*SLOStatus, error) { - return nil, status.Errorf(codes.Unimplemented, "method Status not implemented") +func (UnimplementedSLOServer) GetSLOStatus(context.Context, *v1.Reference) (*SLOStatus, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetSLOStatus not implemented") } func (UnimplementedSLOServer) Preview(context.Context, *CreateSLORequest) (*SLOPreviewResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Preview not implemented") } +func (UnimplementedSLOServer) ListDatasources(context.Context, *emptypb.Empty) (*DatasourceInfo, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListDatasources not implemented") +} func (UnimplementedSLOServer) mustEmbedUnimplementedSLOServer() {} // UnsafeSLOServer may be embedded to opt out of forward compatibility for this service. @@ -429,20 +444,20 @@ func _SLO_ListEvents_Handler(srv interface{}, ctx context.Context, dec func(inte return interceptor(ctx, in, info, handler) } -func _SLO_Status_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { +func _SLO_GetSLOStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(v1.Reference) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(SLOServer).Status(ctx, in) + return srv.(SLOServer).GetSLOStatus(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: SLO_Status_FullMethodName, + FullMethod: SLO_GetSLOStatus_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SLOServer).Status(ctx, req.(*v1.Reference)) + return srv.(SLOServer).GetSLOStatus(ctx, req.(*v1.Reference)) } return interceptor(ctx, in, info, handler) } @@ -465,6 +480,24 @@ func _SLO_Preview_Handler(srv interface{}, ctx context.Context, dec func(interfa return interceptor(ctx, in, info, handler) } +func _SLO_ListDatasources_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(emptypb.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SLOServer).ListDatasources(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: SLO_ListDatasources_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SLOServer).ListDatasources(ctx, req.(*emptypb.Empty)) + } + return interceptor(ctx, in, info, handler) +} + // SLO_ServiceDesc is the grpc.ServiceDesc for SLO service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -513,14 +546,18 @@ var SLO_ServiceDesc = grpc.ServiceDesc{ Handler: _SLO_ListEvents_Handler, }, { - MethodName: "Status", - Handler: _SLO_Status_Handler, + MethodName: "GetSLOStatus", + Handler: _SLO_GetSLOStatus_Handler, }, { MethodName: "Preview", Handler: _SLO_Preview_Handler, }, + { + MethodName: "ListDatasources", + Handler: _SLO_ListDatasources_Handler, + }, }, Streams: []grpc.StreamDesc{}, - Metadata: "github.com/rancher/opni/plugins/slo/apis/slo/slo.proto", + Metadata: "github.com/rancher/opni/pkg/apis/slo/v1/slo.proto", } diff --git a/plugins/slo/apis/slo/validate.go b/pkg/apis/slo/v1/validate.go similarity index 92% rename from plugins/slo/apis/slo/validate.go rename to pkg/apis/slo/v1/validate.go index 68a45bc7e6..38e1e21ab7 100644 --- a/plugins/slo/apis/slo/validate.go +++ b/pkg/apis/slo/v1/validate.go @@ -1,4 +1,4 @@ -package slo +package v1 import ( "time" @@ -26,9 +26,6 @@ func (slo *ServiceLevelObjective) Validate() error { if err != nil { return validation.Error("Passed in sloPeriod string is not a valid prometheus duration") } - if slo.Datasource != shared.MonitoringDatasource && slo.Datasource != shared.LoggingDatasource { - return shared.ErrInvalidDatasource - } if slo.GetName() == "" { return validation.Error("slo name must be set") } @@ -118,9 +115,6 @@ func (l *ListEventsRequest) Validate() error { if l.GetDatasource() == "" { return validation.Error("datasource must be set") } - if l.GetDatasource() != shared.MonitoringDatasource && l.GetDatasource() != shared.LoggingDatasource { - return shared.ErrInvalidDatasource - } if l.GetServiceId() == "" { return validation.Error("serviceId must be set") } diff --git a/pkg/metrics/compat/compat_suite_test.go b/pkg/metrics/compat/compat_suite_test.go new file mode 100644 index 0000000000..e6bf576460 --- /dev/null +++ b/pkg/metrics/compat/compat_suite_test.go @@ -0,0 +1,14 @@ +package compat_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + _ "github.com/rancher/opni/pkg/test/setup" +) + +func TestCompat(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Compat Suite") +} diff --git a/pkg/metrics/compat/unmarshal.go b/pkg/metrics/compat/unmarshal.go index fc4f8bf57c..d7364ad62b 100644 --- a/pkg/metrics/compat/unmarshal.go +++ b/pkg/metrics/compat/unmarshal.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "net/http" + "slices" "github.com/prometheus/common/model" "go.uber.org/zap" @@ -20,7 +21,7 @@ type QueryResult struct { type ErrorType string // struct for unmarshalling from prometheus api responses -type apiResponse struct { +type ApiResponse struct { Status string `json:"status"` Data json.RawMessage `json:"data"` ErrorType ErrorType `json:"errorType"` @@ -85,8 +86,61 @@ func (qr *QueryResult) GetMatrix() (*model.Matrix, error) { } } +func (qr *QueryResult) GetScalar() (*model.Scalar, error) { + switch qr.V.Type() { + case model.ValScalar: + v := *qr.V.(*model.Scalar) + return &v, nil + default: + return nil, fmt.Errorf("cannot unmarshal prometheus response into scalar type") + } +} + +type Sample struct { + Value float64 + // TODO Milliseconds? + Timestamp int64 +} + +func (qr *QueryResult) MapToSamples() []Sample { + res := []Sample{} + switch qr.V.Type() { + case model.ValVector: + v := qr.V.(model.Vector) + for _, sample := range v { + res = append(res, Sample{ + Value: float64(sample.Value), + Timestamp: int64(sample.Timestamp), + }) + } + case model.ValScalar: + v := qr.V.(*model.Scalar) + res = append(res, Sample{ + Value: float64(v.Value), + Timestamp: int64(v.Timestamp), + }) + case model.ValMatrix: + sampleStream := qr.V.(model.Matrix) + for _, sample := range sampleStream { + for _, s := range sample.Values { + res = append(res, Sample{ + Value: float64(s.Value), + Timestamp: int64(s.Timestamp), + }) + } + } + default: + panic("bug: unreachable code") + } + + slices.SortFunc(res, func(i, j Sample) int { + return int(i.Timestamp - j.Timestamp) + }) + return res +} + func UnmarshalPrometheusResponse(data []byte) (*QueryResult, error) { - var a apiResponse + var a ApiResponse var q QueryResult if err := json.Unmarshal(data, &a); err != nil { @@ -130,15 +184,6 @@ type Response struct { Warnings []string `json:"warnings,omitempty"` } -func unmarshallPrometheusWebResponseData(data []byte) (*Response, error) { - var r Response - err := json.Unmarshal(data, &r) - if err != nil { - return nil, err - } - return &r, nil -} - func UnmarshallPrometheusWebResponse(resp *http.Response, _ *zap.SugaredLogger) (*Response, error) { var val *Response err := json.NewDecoder(resp.Body).Decode(&val) diff --git a/pkg/metrics/compat/unmarshal_test.go b/pkg/metrics/compat/unmarshal_test.go new file mode 100644 index 0000000000..e3f743145a --- /dev/null +++ b/pkg/metrics/compat/unmarshal_test.go @@ -0,0 +1,198 @@ +package compat_test + +import ( + "encoding/json" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/prometheus/common/model" + "github.com/rancher/opni/pkg/metrics/compat" +) + +var _ = Describe("Prometheus Query response unmarshalling", Label("unit"), func() { + It("should unmarshal scalar responses", func() { + scalarData := model.Scalar{ + Value: 1, + Timestamp: model.Time(1), + } + + scalarDataWrapper := struct { + Type model.ValueType `json:"resultType"` + Result json.RawMessage `json:"result"` + }{} + scalarDataWrapper.Type = model.ValScalar + scalarDataBytes, err := json.Marshal(scalarData) + Expect(err).To(Succeed()) + scalarDataWrapper.Result = scalarDataBytes + + scalarDataWrapperBytes, err := json.Marshal(scalarDataWrapper) + Expect(err).To(Succeed()) + + promResp := compat.ApiResponse{ + Status: "success", + Data: scalarDataWrapperBytes, + ErrorType: "", + Error: "", + Warnings: []string{}, + } + + promRespData, err := json.Marshal(promResp) + Expect(err).To(Succeed()) + + qr, err := compat.UnmarshalPrometheusResponse(promRespData) + Expect(err).To(Succeed()) + Expect(qr.Type).To(Equal(model.ValScalar)) + Expect(qr.V).To(Equal(&scalarData)) + + vector, err := qr.GetVector() + Expect(err).NotTo(Succeed()) + Expect(vector).To(BeNil()) + + matrix, err := qr.GetMatrix() + Expect(err).NotTo(Succeed()) + Expect(matrix).To(BeNil()) + + scalar, err := qr.GetScalar() + Expect(err).To(Succeed()) + Expect(scalar.Value).To(Equal(scalarData.Value)) + Expect(scalar.Timestamp).To(Equal(scalarData.Timestamp)) + + samples := qr.MapToSamples() + Expect(samples).To(HaveLen(1)) + Expect(samples[0].Value).To(Equal(float64(scalarData.Value))) + Expect(samples[0].Timestamp).To(Equal(int64(scalarData.Timestamp))) + }) + + It("should unmarshal vector responses", func() { + vectorData := model.Vector{ + { + Value: 1, + Timestamp: model.Time(1), + }, + { + Value: 2, + Timestamp: model.Time(0), + }, + } + + dataWrapper := struct { + Type model.ValueType `json:"resultType"` + Result json.RawMessage `json:"result"` + }{} + dataWrapper.Type = model.ValVector + scalarDataBytes, err := json.Marshal(vectorData) + Expect(err).To(Succeed()) + dataWrapper.Result = scalarDataBytes + + scalarDataWrapperBytes, err := json.Marshal(dataWrapper) + Expect(err).To(Succeed()) + + promResp := compat.ApiResponse{ + Status: "success", + Data: scalarDataWrapperBytes, + ErrorType: "", + Error: "", + Warnings: []string{}, + } + + promRespData, err := json.Marshal(promResp) + Expect(err).To(Succeed()) + + qr, err := compat.UnmarshalPrometheusResponse(promRespData) + Expect(err).To(Succeed()) + Expect(qr.Type).To(Equal(model.ValVector)) + + scalar, err := qr.GetScalar() + Expect(err).NotTo(Succeed()) + Expect(scalar).To(BeNil()) + + matrix, err := qr.GetMatrix() + Expect(err).NotTo(Succeed()) + Expect(matrix).To(BeNil()) + + vector, err := qr.GetVector() + Expect(err).To(Succeed()) + Expect(vector).NotTo(BeNil()) + + samples := qr.MapToSamples() + Expect(samples).To(HaveLen(2)) + Expect(samples[0].Value).To(Equal(float64(vectorData[1].Value))) + Expect(samples[0].Timestamp).To(Equal(int64(vectorData[1].Timestamp))) + Expect(samples[1].Value).To(Equal(float64(vectorData[0].Value))) + Expect(samples[1].Timestamp).To(Equal(int64(vectorData[0].Timestamp))) + }) + + It("should unmarshal matrix responses", func() { + matrixData := model.Matrix{ + { + Values: []model.SamplePair{ + { + Value: 1, + Timestamp: model.Time(2), + }, + + { + Value: 2, + Timestamp: model.Time(0), + }, + }, + }, + { + Values: []model.SamplePair{ + { + Value: 3, + Timestamp: model.Time(1), + }, + }, + }, + } + + dataWrapper := struct { + Type model.ValueType `json:"resultType"` + Result json.RawMessage `json:"result"` + }{} + dataWrapper.Type = model.ValMatrix + scalarDataBytes, err := json.Marshal(matrixData) + Expect(err).To(Succeed()) + dataWrapper.Result = scalarDataBytes + + scalarDataWrapperBytes, err := json.Marshal(dataWrapper) + Expect(err).To(Succeed()) + + promResp := compat.ApiResponse{ + Status: "success", + Data: scalarDataWrapperBytes, + ErrorType: "", + Error: "", + Warnings: []string{}, + } + + promRespData, err := json.Marshal(promResp) + Expect(err).To(Succeed()) + + qr, err := compat.UnmarshalPrometheusResponse(promRespData) + Expect(err).To(Succeed()) + Expect(qr.Type).To(Equal(model.ValMatrix)) + + scalar, err := qr.GetScalar() + Expect(err).NotTo(Succeed()) + Expect(scalar).To(BeNil()) + + vector, err := qr.GetVector() + Expect(err).NotTo(Succeed()) + Expect(vector).To(BeNil()) + + matrix, err := qr.GetMatrix() + Expect(err).To(Succeed()) + Expect(matrix).NotTo(BeNil()) + + samples := qr.MapToSamples() + Expect(samples).To(HaveLen(3)) + Expect(samples[0].Value).To(Equal(float64(2))) + Expect(samples[0].Timestamp).To(Equal(int64(0))) + Expect(samples[1].Value).To(Equal(float64(3))) + Expect(samples[1].Timestamp).To(Equal(int64(1))) + Expect(samples[2].Value).To(Equal(float64(1))) + Expect(samples[2].Timestamp).To(Equal(int64(2))) + }) +}) diff --git a/pkg/slo/backend/backend_suite_test.go b/pkg/slo/backend/backend_suite_test.go new file mode 100644 index 0000000000..4948312ed9 --- /dev/null +++ b/pkg/slo/backend/backend_suite_test.go @@ -0,0 +1,14 @@ +package backend_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + _ "github.com/rancher/opni/pkg/test/setup" +) + +func TestBackend(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Backend Suite") +} diff --git a/pkg/slo/backend/event_test.go b/pkg/slo/backend/event_test.go new file mode 100644 index 0000000000..baddfc669d --- /dev/null +++ b/pkg/slo/backend/event_test.go @@ -0,0 +1,122 @@ +package backend_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/rancher/opni/pkg/slo/backend" +) + +var _ = Describe("Event matching", Label("unit"), func() { + When("we receive event data from the user", func() { + It("should match empty total events to all events", func() { + goodEvents := []*slov1.Event{ + { + Key: "code", + Vals: []string{"200"}, + }, + } + totalEvents := []*slov1.Event{} + g, t := backend.MatchEventsOnMetric(goodEvents, totalEvents) + Expect(g).To(Equal([]*slov1.Event{ + { + Key: "code", + Vals: []string{"200"}, + }, + })) + Expect(t).To(Equal([]*slov1.Event{})) + }) + + It("should include good label filters in matching total event filters ", func() { + goodEvents := []*slov1.Event{ + { + Key: "code", + Vals: []string{"200"}, + }, + } + totalEvents := []*slov1.Event{ + { + Key: "code", + Vals: []string{"500", "503"}, + }, + } + g, t := backend.MatchEventsOnMetric(goodEvents, totalEvents) + Expect(g).To(Equal( + []*slov1.Event{ + { + Key: "code", + Vals: []string{"200"}, + }, + }, + )) + Expect(t).To(Equal( + []*slov1.Event{ + { + Key: "code", + Vals: []string{"200", "500", "503"}, + }, + }, + )) + }) + + It("should apply missing total label filters to good filters", func() { + goodEvents := []*slov1.Event{ + { + Key: "code", + Vals: []string{"200"}, + }, + } + totalEvents := []*slov1.Event{ + { + Key: "handler", + Vals: []string{"/ready"}, + }, + } + g, t := backend.MatchEventsOnMetric(goodEvents, totalEvents) + + Expect(g).To(Equal( + []*slov1.Event{ + { + Key: "code", + Vals: []string{"200"}, + }, + { + Key: "handler", + Vals: []string{"/ready"}, + }, + }, + )) + Expect(t).To(Equal( + []*slov1.Event{ + { + Key: "handler", + Vals: []string{"/ready"}, + }, + }, + )) + }) + + It("should sanitize invalid inputs by omitting them", func() { + goodEvents := []*slov1.Event{ + { + Key: "", + Vals: []string{"200"}, + }, + } + totalEvents := []*slov1.Event{ + { + Key: "", + Vals: []string{"500", "503"}, + }, + { + Key: "handler", + Vals: nil, + }, + } + + g, t := backend.MatchEventsOnMetric(goodEvents, totalEvents) + Expect(g).To(Equal([]*slov1.Event{})) + Expect(t).To(Equal([]*slov1.Event{})) + }) + }) +}) diff --git a/pkg/slo/backend/metrics/backend.go b/pkg/slo/backend/metrics/backend.go new file mode 100644 index 0000000000..5a3c06ad37 --- /dev/null +++ b/pkg/slo/backend/metrics/backend.go @@ -0,0 +1,88 @@ +package metrics + +import ( + "context" + "fmt" + + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/rancher/opni/pkg/slo/backend" + "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" + "github.com/tidwall/gjson" +) + +type MetricsBackend struct { + *MetricsBackendProvider +} + +func NewBackend(p *MetricsBackendProvider) *MetricsBackend { + return &MetricsBackend{ + MetricsBackendProvider: p, + } +} + +var _ backend.ServiceBackend = &MetricsBackend{} + +func (m *MetricsBackend) ListServices(ctx context.Context, req *slov1.ListServicesRequest) (*slov1.ServiceList, error) { + services := &slov1.ServiceList{} + discoveryQuery := `group by (job) ({__name__!=""})` + // TODO : this should probably be a metrics API + resp, err := m.adminClient.Get().Query( + ctx, + &cortexadmin.QueryRequest{ + Tenants: []string{req.GetClusterId()}, + Query: discoveryQuery, + }) + if err != nil { + return nil, err + } + result := gjson.Get(string(resp.Data), "data.result.#.metric.job") + if !result.Exists() { + return nil, fmt.Errorf("could not convert prometheus service discovery to json") + } + for _, v := range result.Array() { + services.Items = append(services.Items, &slov1.Service{ + ClusterId: req.GetClusterId(), + ServiceId: v.String(), + }) + } + return services, nil +} +func (m *MetricsBackend) ListMetrics(ctx context.Context, req *slov1.ListMetricsRequest) (*slov1.MetricGroupList, error) { + resp, err := m.adminClient.Get().GetSeriesMetrics( + ctx, + &cortexadmin.SeriesRequest{ + Tenant: req.GetClusterId(), + JobId: req.GetServiceId(), + }, + ) + if err != nil { + return nil, err + } + return scoredLabels(resp, m.filters), nil +} + +func (m *MetricsBackend) ListEvents(ctx context.Context, req *slov1.ListEventsRequest) (*slov1.EventList, error) { + events := &slov1.EventList{ + Items: []*slov1.Event{}, + } + + resp, err := m.adminClient.Get().GetMetricLabelSets( + ctx, + &cortexadmin.LabelRequest{ + Tenant: req.GetClusterId(), + JobId: req.GetServiceId(), + MetricName: req.GetMetricId(), + }, + ) + if err != nil { + return nil, err + } + for _, item := range resp.GetItems() { + events.Items = append(events.Items, &slov1.Event{ + Key: item.GetName(), + Vals: item.GetItems(), + }) + } + + return events, nil +} diff --git a/pkg/slo/backend/metrics/backend_provider.go b/pkg/slo/backend/metrics/backend_provider.go new file mode 100644 index 0000000000..6a62a87e49 --- /dev/null +++ b/pkg/slo/backend/metrics/backend_provider.go @@ -0,0 +1,184 @@ +package metrics + +import ( + "embed" + "io/fs" + "path/filepath" + "regexp" + "sync" + + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/rancher/opni/pkg/util/future" + "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" + "github.com/samber/lo" + "go.uber.org/zap" + "gopkg.in/yaml.v3" +) + +//go:embed metricgroups/*.yaml +var metricGroups embed.FS + +//go:embed servicegroups/*.yaml +var serviceGroups embed.FS + +// map of directory names to their embed.FS +var EnabledFilters = map[string]embed.FS{"metricgroups": metricGroups, "servicegroups": serviceGroups} + +type MetricsBackendProvider struct { + lg *zap.SugaredLogger + filters []Filter + adminClient future.Future[cortexadmin.CortexAdminClient] +} + +func NewProvider( + lg *zap.SugaredLogger, +) *MetricsBackendProvider { + return &MetricsBackendProvider{ + lg: lg, + adminClient: future.New[cortexadmin.CortexAdminClient](), + } +} + +func (m *MetricsBackendProvider) Initialize(cl cortexadmin.CortexAdminClient) { + m.adminClient.Set(cl) + m.filters = m.constructFilters() +} + +func ApplyFiltersToMetricEvents(seriesInfo *cortexadmin.SeriesInfoList, filters []Filter) *slov1.MetricGroupList { + return scoredLabels(seriesInfo, filters) +} + +func GetGroupConfigsFromEmbed(dirName string, dir embed.FS) []Filter { + var res []Filter + var mu sync.Mutex + fsys := fs.FS(dir) + yamlFs, err := fs.Sub(fsys, dirName) + if err != nil { + return []Filter{} + } + fs.WalkDir(yamlFs, ".", func(pathStr string, d fs.DirEntry, err error) error { + if !d.IsDir() { + fBytes, err := fs.ReadFile(dir, filepath.Join(dirName, pathStr)) + if err != nil { + panic(err) + } + var f Filter + err = yaml.Unmarshal(fBytes, &f) + if err == nil { + mu.Lock() + res = append(res, f) + mu.Unlock() + } + } + return nil + }) + return res +} + +func scoredLabels(seriesInfo *cortexadmin.SeriesInfoList, filters []Filter) *slov1.MetricGroupList { + + seriesNameToScoreGroup := map[string]map[string]int{} + + seriesNameToSeries := lo.Associate(seriesInfo.GetItems(), func(series *cortexadmin.SeriesInfo) (string, *cortexadmin.SeriesInfo) { + return series.GetSeriesName(), series + }) + + for _, series := range seriesInfo.GetItems() { + seriesName := series.GetSeriesName() + for _, groupname := range filters { + for _, matchFilter := range groupname.Filters { + if matchFilter.Value.MatchString(series.GetSeriesName()) { + if _, ok := seriesNameToScoreGroup[seriesName]; !ok { + seriesNameToScoreGroup[seriesName] = map[string]int{} + } + seriesNameToScoreGroup[seriesName][groupname.Name] += matchFilter.Score + } + } + + for _, ignoreFilter := range groupname.Ignore { + if ignoreFilter.Value.MatchString(series.GetSeriesName()) { + if _, ok := seriesNameToScoreGroup[seriesName]; !ok { + seriesNameToScoreGroup[seriesName] = map[string]int{} + } + seriesNameToScoreGroup[seriesName][groupname.Name] -= ignoreFilter.Score + } + } + } + } + groupedMetrics := &slov1.MetricGroupList{ + GroupNameToMetrics: map[string]*slov1.MetricList{}, + } + for seriesName, data := range seriesNameToSeries { + groupName := "" + maxScore := 0 + + if vals, ok := seriesNameToScoreGroup[seriesName]; ok { + for group, score := range vals { + if score > maxScore { + maxScore = score + groupName = group + } + } + } + + if maxScore <= 0 { + groupName = "other metrics" + } + if _, ok := groupedMetrics.GroupNameToMetrics[groupName]; !ok { + groupedMetrics.GroupNameToMetrics[groupName] = &slov1.MetricList{} + } + groupedMetrics.GroupNameToMetrics[groupName].Items = append(groupedMetrics.GroupNameToMetrics[groupName].Items, &slov1.Metric{ + Id: seriesName, + Metadata: &slov1.MetricMetadata{ + Description: data.Metadata.GetDescription(), + Unit: data.Metadata.GetUnit(), + Type: data.Metadata.GetType(), + }, + }) + } + return groupedMetrics +} + +func (m *MetricsBackendProvider) constructFilters() []Filter { + res := []Filter{} + for dirName, embedFs := range EnabledFilters { + filters := GetGroupConfigsFromEmbed(dirName, embedFs) + res = append(res, filters...) + } + return res +} + +type Filter struct { + Name string `yaml:"name"` + Filters []FilterValue `yaml:"filters"` + Ignore []FilterValue `yaml:"ignore"` +} + +type FilterValue struct { + Value Regexp `yaml:"value"` + Score int `yaml:"score"` +} + +type Regexp struct { + *regexp.Regexp +} + +// UnmarshalText unmarshals json into a regexp.Regexp +func (r *Regexp) UnmarshalText(b []byte) error { + regex, err := regexp.Compile(string(b)) + if err != nil { + return err + } + + r.Regexp = regex + + return nil +} + +// MarshalText marshals regexp.Regexp as string +func (r *Regexp) MarshalText() ([]byte, error) { + if r.Regexp != nil { + return []byte(r.Regexp.String()), nil + } + return nil, nil +} diff --git a/pkg/slo/backend/metrics/backend_test.go b/pkg/slo/backend/metrics/backend_test.go new file mode 100644 index 0000000000..b7362a6983 --- /dev/null +++ b/pkg/slo/backend/metrics/backend_test.go @@ -0,0 +1,96 @@ +package metrics_test + +import ( + "context" + + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/rancher/opni/pkg/logger" + "github.com/rancher/opni/pkg/slo/backend/metrics" + cortexadmin_mock "github.com/rancher/opni/pkg/test/mock/cortexadmin" + "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" +) + +var _ = Describe("Metrics SLO Service backend", Label("unit"), Ordered, func() { + var mb *metrics.MetricsBackend + BeforeAll(func() { + ctrl := gomock.NewController(GinkgoT()) + mockClient := cortexadmin_mock.NewMockCortexAdminClient(ctrl) + + mockClient.EXPECT(). + GetSeriesMetrics(gomock.Any(), gomock.Any()). + Return(&cortexadmin.SeriesInfoList{ + Items: []*cortexadmin.SeriesInfo{ + { + SeriesName: "test", + }, + }, + }, nil).AnyTimes() + + mockClient.EXPECT(). + GetMetricLabelSets(gomock.Any(), gomock.Any()). + Return(&cortexadmin.MetricLabels{ + Items: []*cortexadmin.LabelSet{ + { + Name: "test-label", + Items: []string{"host_name"}, + }, + }, + }, nil).AnyTimes() + + m := metrics.NewProvider(logger.NewPluginLogger().Named("slo")) + m.Initialize(mockClient) + mb = metrics.NewBackend(m) + }) + When("we we use the metrics service backend", func() { + XIt("should list discoverable services", func() { //FIXME: TODO: + svcList, err := mb.ListServices( + context.TODO(), + &slov1.ListServicesRequest{ + Datasource: "metrics", + ClusterId: "test", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(svcList.Items).To(ContainElement(&slov1.Service{ + ClusterId: "test", + ServiceId: "test", + })) + }) + + XIt("should list metrics based on services", func() { + metricList, err := mb.ListMetrics( + context.TODO(), + &slov1.ListMetricsRequest{ + Datasource: "metrics", + ClusterId: "test", + ServiceId: "test", + }, + ) + Expect(err).NotTo(HaveOccurred()) + Expect(metricList).To(Equal(&slov1.MetricGroupList{ + GroupNameToMetrics: map[string]*slov1.MetricList{}, + })) + }) + + It("should list events based on metrics and services", func() { + eventList, err := mb.ListEvents(context.TODO(), + &slov1.ListEventsRequest{ + Datasource: "metrics", + ClusterId: "test", + ServiceId: "test", + MetricId: "test", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(eventList).To(Equal(&slov1.EventList{ + Items: []*slov1.Event{ + { + Key: "test-label", + Vals: []string{"host_name"}, + }, + }, + })) + }) + }) +}) diff --git a/pkg/slo/backend/metrics/cortex.go b/pkg/slo/backend/metrics/cortex.go new file mode 100644 index 0000000000..2759a187c4 --- /dev/null +++ b/pkg/slo/backend/metrics/cortex.go @@ -0,0 +1,9 @@ +package metrics + +import ( + "github.com/prometheus/prometheus/model/rulefmt" +) + +type RuleGroupBuilder interface { + AsRuleGroup() (rulefmt.RuleGroup, error) +} diff --git a/pkg/slo/backend/metrics/filter_test.go b/pkg/slo/backend/metrics/filter_test.go new file mode 100644 index 0000000000..d04c376201 --- /dev/null +++ b/pkg/slo/backend/metrics/filter_test.go @@ -0,0 +1,73 @@ +package metrics_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/rancher/opni/pkg/slo/backend/metrics" + "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" +) + +var _ = Describe("SLO Filter tests", Ordered, Label("unit"), func() { + When("We use SLO filters", func() { + var filters []metrics.Filter + It("should get parse them from our embedded directory definitions", func() { + filters = make([]metrics.Filter, 0) + for dirName, embedFs := range metrics.EnabledFilters { + flts := metrics.GetGroupConfigsFromEmbed(dirName, embedFs) + filters = append(filters, flts...) + } + for _, filter := range filters { + Expect(filter.Name).NotTo(Equal("")) + Expect(filter.Filters).NotTo(BeEmpty()) + } + Expect(filters).To(HaveLen(6)) + }) + + It("Should be able to score the events based on the filters", func() { + // array of prom metrics : label name -> label vals + series := &cortexadmin.SeriesInfoList{ + Items: []*cortexadmin.SeriesInfo{ + { + SeriesName: "go_gc_duration_seconds", + }, + { + SeriesName: "jvm_something", + }, + { + SeriesName: "jvm_something_else", + }, + { + SeriesName: "uptime_seconds", + }, + { + SeriesName: "kube-proxy", + }, + { + SeriesName: "apiserver_something", + }, + { + SeriesName: "request_duration_seconds", + }, + { + SeriesName: "cpu_usage_seconds_total", + }, + { + SeriesName: ".*kube-prometheus-operator.*", + }, + }, + } + + filteredGroupsResp := metrics.ApplyFiltersToMetricEvents(series, filters) + Expect(filteredGroupsResp).NotTo(BeNil()) + + filteredGroups := filteredGroupsResp.GetGroupNameToMetrics() + + Expect(filteredGroups).To(HaveKey("golang metrics")) + Expect(filteredGroups).To(HaveKey("jvm metrics")) + Expect(filteredGroups).To(HaveKey("kubernetes metrics")) + Expect(filteredGroups).To(HaveKey("network metrics")) + Expect(filteredGroups).To(HaveKey("compute metrics")) + Expect(filteredGroups).To(HaveKey("other metrics")) + }) + }) +}) diff --git a/pkg/slo/backend/metrics/manager.go b/pkg/slo/backend/metrics/manager.go new file mode 100644 index 0000000000..828f12593d --- /dev/null +++ b/pkg/slo/backend/metrics/manager.go @@ -0,0 +1,375 @@ +package metrics + +import ( + "context" + "sync" + "time" + + "github.com/google/uuid" + prommodel "github.com/prometheus/common/model" + corev1 "github.com/rancher/opni/pkg/apis/core/v1" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/rancher/opni/pkg/metrics/compat" + "github.com/rancher/opni/pkg/slo/backend" + "github.com/rancher/opni/pkg/util" + "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" + "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" + "gopkg.in/yaml.v3" +) + +const ( + sloRuleNamespace = "slo" +) + +type MetricsSLOStoreProvider struct { + lg *zap.SugaredLogger + adminClient cortexadmin.CortexAdminClient +} + +func NewMetricsSLOStoreProvider( + lg *zap.SugaredLogger, +) *MetricsSLOStoreProvider { + return &MetricsSLOStoreProvider{ + lg: lg, + } +} + +func (m *MetricsSLOStoreProvider) Initialize(adminClient cortexadmin.CortexAdminClient) { + m.adminClient = adminClient +} + +type MetricsSLOStore struct { + *MetricsSLOStoreProvider + + // backend.ServiceBackend +} + +var _ backend.SLOStore = (*MetricsSLOStore)(nil) + +func NewMetricsSLOStore( + provider *MetricsSLOStoreProvider, +) *MetricsSLOStore { + return &MetricsSLOStore{ + MetricsSLOStoreProvider: provider, + } +} + +func (m *MetricsSLOStore) Create(ctx context.Context, req *slov1.CreateSLORequest) (*corev1.Reference, error) { + slo := backend.CreateSLORequestToStruct(req) + sloGen, err := NewSLOGenerator(*slo) + if err != nil { + return nil, err + } + ruleGroup, err := sloGen.AsRuleGroup() + if err != nil { + return nil, err + } + yamlBytes, err := yaml.Marshal(ruleGroup) + if err != nil { + return nil, err + } + if _, err := m.adminClient.LoadRules(ctx, &cortexadmin.LoadRuleRequest{ + Namespace: sloRuleNamespace, + YamlContent: yamlBytes, + ClusterId: req.Slo.ClusterId, + }); err != nil { + return nil, err + } + + return &corev1.Reference{ + Id: slo.GetId(), + }, nil +} +func (m *MetricsSLOStore) Update(ctx context.Context, incoming, existing *slov1.SLOData) (*slov1.SLOData, error) { + incomingSLO := backend.SLODataToStruct(incoming) + existingSLO := backend.SLODataToStruct(existing) + + yamlBytes, err := yaml.Marshal(incomingSLO) + if err != nil { + return nil, err + } + if _, err := m.adminClient.LoadRules(ctx, &cortexadmin.LoadRuleRequest{ + Namespace: sloRuleNamespace, + YamlContent: yamlBytes, + ClusterId: incoming.SLO.ClusterId, + }); err != nil { + return nil, err + } + + if incoming.SLO.ClusterId != existing.SLO.ClusterId { + m.adminClient.DeleteRule(ctx, &cortexadmin.DeleteRuleRequest{ + Namespace: sloRuleNamespace, + GroupName: existingSLO.GetId(), + ClusterId: existing.SLO.ClusterId, + }) + } + return incoming, nil +} +func (m *MetricsSLOStore) Delete(ctx context.Context, existing *slov1.SLOData) error { + slo := backend.SLODataToStruct(existing) + _, err := m.adminClient.DeleteRule(ctx, &cortexadmin.DeleteRuleRequest{ + Namespace: sloRuleNamespace, + GroupName: slo.GetId(), + ClusterId: existing.SLO.ClusterId, + }) + return err +} +func (m *MetricsSLOStore) Clone(ctx context.Context, clone *slov1.SLOData) (*corev1.Reference, *slov1.SLOData, error) { + clonedData := util.ProtoClone(clone) + sloData := clonedData.GetSLO() + clonedSLO := backend.SLODataToStruct(clonedData) + clonedSLO.SetId(uuid.New().String()) + clonedSLO.SetName(sloData.GetName() + "-clone") + yamlBytes, err := yaml.Marshal(clonedSLO) + if err != nil { + return nil, nil, err + } + if _, err := m.adminClient.LoadRules(ctx, &cortexadmin.LoadRuleRequest{ + Namespace: sloRuleNamespace, + YamlContent: yamlBytes, + ClusterId: sloData.ClusterId, + }); err != nil { + return nil, nil, err + } + return &corev1.Reference{ + Id: clonedSLO.GetId(), + }, clonedData, nil +} +func (m *MetricsSLOStore) MultiClusterClone( + ctx context.Context, + slo *slov1.SLOData, + toClusters []*corev1.Reference, +) ([]*corev1.Reference, []*slov1.SLOData, []error) { + errors := make([]error, len(toClusters)) + cloned := make([]*slov1.SLOData, len(toClusters)) + refs := make([]*corev1.Reference, len(toClusters)) + var wg sync.WaitGroup + var mu sync.Mutex + for i, cluster := range toClusters { + cluster := cluster + i := i + wg.Add(1) + go func() { + defer wg.Done() + clone, _, err := m.Clone(ctx, slo) + mu.Lock() + defer mu.Unlock() + errors[i] = err + if err != nil { + return + } + cloned[i] = util.ProtoClone(slo) + cloned[i].SLO.ClusterId = cluster.Id + refs[i] = clone + }() + } + wg.Wait() + return refs, cloned, errors + +} + +// 1. Check if the rule is loaded +// 2. Check SLI, if there is no data, return NoData +// 3. Check the error budget remaining, if exceeded, return Breaching +// 4. Check the MWMB burn rates, if they are elevated, return Warning +// 5. Otherwise, return Ok +func (m *MetricsSLOStore) Status(ctx context.Context, slo *slov1.SLOData) (*slov1.SLOStatus, error) { + clusterId := slo.SLO.ClusterId + // 1. + _, err := m.adminClient.GetRule(ctx, &cortexadmin.GetRuleRequest{ + Namespace: sloRuleNamespace, + ClusterId: clusterId, + GroupName: slo.GetId(), + }) + if st, ok := status.FromError(err); ok && st.Code() == codes.NotFound { + return &slov1.SLOStatus{ + State: slov1.SLOStatusState_Creating, + }, nil + } else if err != nil { + return nil, err + } + + sloGen, err := NewSLOGenerator(*backend.SLODataToStruct(slo)) + if err != nil { + return nil, err + } + + // 2. + dur, err := prommodel.ParseDuration(slo.SLO.SloPeriod) + if err != nil { + return nil, err + } + sliExpr := sloGen.SLI(dur).Expr() + rawSLIRes, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ + Tenants: []string{clusterId}, + Query: sliExpr, + }) + if err != nil { + return nil, err + } + qrSLI, err := compat.UnmarshalPrometheusResponse(rawSLIRes.Data) + if err != nil { + return nil, err + } + if len(qrSLI.MapToSamples()) == 0 { + return &slov1.SLOStatus{ + State: slov1.SLOStatusState_NoData, + }, nil + } + + // 3. + remainingBudget := sloGen.ErrorBudgetRemaining().Expr() + rawRemBudget, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ + Tenants: []string{clusterId}, + Query: remainingBudget, + }) + if err != nil { + return nil, err + } + qrBudget, err := compat.UnmarshalPrometheusResponse(rawRemBudget.Data) + if err != nil { + return nil, err + } + samples := qrBudget.MapToSamples() + if len(samples) == 0 { + return &slov1.SLOStatus{ + State: slov1.SLOStatusState_NoData, + }, nil + } + lastErrorBudget := samples[len(samples)-1].Value + if lastErrorBudget < 0 { + return &slov1.SLOStatus{ + State: slov1.SLOStatusState_Breaching, + }, nil + } + + // 4. + pageAlert := sloGen.PageAlert().Expr() + rawPageAlert, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ + Tenants: []string{clusterId}, + Query: pageAlert, + }) + if err != nil { + return nil, err + } + qrPage, err := compat.UnmarshalPrometheusResponse(rawPageAlert.Data) + if err != nil { + return nil, err + } + samples = qrPage.MapToSamples() + if len(samples) != 0 { + lastPageAlert := samples[len(samples)-1].Value + if lastPageAlert > 0 { + return &slov1.SLOStatus{ + State: slov1.SLOStatusState_Warning, + }, nil + } + } + + ticketAlert := sloGen.TicketAlert().Expr() + rawTicketAlert, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ + Tenants: []string{clusterId}, + Query: ticketAlert, + }) + if err != nil { + return nil, err + } + qrTicket, err := compat.UnmarshalPrometheusResponse(rawTicketAlert.Data) + if err != nil { + return nil, err + } + samples = qrTicket.MapToSamples() + if len(samples) != 0 { + lastPageAlert := samples[len(samples)-1].Value + if lastPageAlert > 0 { + return &slov1.SLOStatus{ + State: slov1.SLOStatusState_Warning, + }, nil + } + } + return &slov1.SLOStatus{ + State: slov1.SLOStatusState_Ok, + }, nil +} +func (m *MetricsSLOStore) Preview(ctx context.Context, slo *slov1.CreateSLORequest) (*slov1.SLOPreviewResponse, error) { + sloDef := backend.SLODataToStruct(&slov1.SLOData{ + Id: "preview-" + uuid.New().String(), + SLO: slo.Slo, + }) + + sloGen, err := NewSLOGenerator(*sloDef) + if err != nil { + return nil, err + } + + period, err := prommodel.ParseDuration(slo.Slo.SloPeriod) + if err != nil { + return nil, err + } + sli := sloGen.SLI(period).Expr() + cur := time.Now() + startTs, endTs := cur.Add(time.Duration(-period)), cur + numSteps := 250 + step := time.Duration(endTs.Sub(startTs).Seconds()/float64(numSteps)) * time.Second + if step < time.Second { + step = time.Second + } + rawSLI, err := m.adminClient.QueryRange(ctx, &cortexadmin.QueryRangeRequest{ + Tenants: []string{slo.Slo.ClusterId}, + Query: sli, + Start: timestamppb.New(startTs), + End: timestamppb.New(endTs), + Step: durationpb.New(step), + }) + if err != nil { + return nil, err + } + + qrSLI, err := compat.UnmarshalPrometheusResponse(rawSLI.Data) + if err != nil { + return nil, err + } + + samples := qrSLI.MapToSamples() + + plotVector := &slov1.PlotVector{ + Objective: slo.Slo.Target.Value, + Items: make([]*slov1.DataPoint, len(samples)), + } + + for i := 0; i < len(samples); i++ { + plotVector.Items[i] = &slov1.DataPoint{ + Sli: samples[i].Value, + Timestamp: timestamppb.Now(), + } + } + + // TODO: detect alert windows + // pageAlert := sloGen.PageAlert().Expr() + // rawPageAlert, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ + // Tenants: []string{slo.Slo.ClusterId}, + // Query: pageAlert, + // }) + // fmt.Println(rawPageAlert) + // if err != nil { + // return nil, err + // } + // // TODO : check values + // ticketAlert := sloGen.TicketAlert().Expr() + // rawTicketAlert, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ + // Tenants: []string{slo.Slo.ClusterId}, + // Query: ticketAlert, + // }) + // fmt.Println(rawTicketAlert) + // if err != nil { + // return nil, err + // } + + return &slov1.SLOPreviewResponse{ + PlotVector: plotVector, + }, nil +} diff --git a/pkg/slo/backend/metrics/manager_test.go b/pkg/slo/backend/metrics/manager_test.go new file mode 100644 index 0000000000..37b71b3664 --- /dev/null +++ b/pkg/slo/backend/metrics/manager_test.go @@ -0,0 +1,162 @@ +package metrics_test + +import ( + "context" + "errors" + "time" + + "github.com/golang/mock/gomock" + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "github.com/rancher/opni/pkg/apis/core/v1" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/rancher/opni/pkg/logger" + "github.com/rancher/opni/pkg/slo/backend/metrics" + cortexadmin_mock "github.com/rancher/opni/pkg/test/mock/cortexadmin" + "github.com/rancher/opni/pkg/util" + "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/emptypb" +) + +// TODO : maybe move this to a conformance test +var _ = Describe("SLO Metrics store", Ordered, Label("unit"), func() { + var metricsStore *metrics.MetricsSLOStore + BeforeAll(func() { + ctrl := gomock.NewController(GinkgoT()) + + mockClient := cortexadmin_mock.NewMockCortexAdminClient(ctrl) + + mockClient.EXPECT().LoadRules(gomock.Any(), gomock.Any()).Return(&emptypb.Empty{}, nil).AnyTimes() + + mockClient.EXPECT().DeleteRule(gomock.Any(), gomock.Any()).Return(&emptypb.Empty{}, nil).AnyTimes() + + mockClient.EXPECT().GetRule(gomock.Any(), gomock.Any()).Return(nil, status.Error(codes.NotFound, "rule not found")).AnyTimes() + + mockClient.EXPECT().Query(gomock.Any(), gomock.Any()).Return(&cortexadmin.QueryResponse{ + Data: []byte{}, + }, nil).AnyTimes() + + lg := logger.NewPluginLogger().Named("slo") + provider := metrics.NewMetricsSLOStoreProvider(lg) + provider.Initialize(mockClient) + metricsStore = metrics.NewMetricsSLOStore( + provider, + ) + }) + + When("we use the mocked metrics store", func() { + It("should be able to create metrics SLOs", func() { + ref, err := metricsStore.Create(context.TODO(), &slov1.CreateSLORequest{ + Slo: util.ProtoClone(mockSLO), + }) + Expect(err).To(Succeed()) + Expect(ref).NotTo(BeNil()) + }) + + It("should be able to update metrics SLOs", func() { + ref, err := metricsStore.Update( + context.TODO(), + &slov1.SLOData{ + Id: uuid.New().String(), + SLO: util.ProtoClone(mockSLO), + }, + &slov1.SLOData{ + Id: uuid.New().String(), + SLO: util.ProtoClone(mockSLO), + }, + ) + Expect(err).To(Succeed()) + Expect(ref).NotTo(BeNil()) + }) + + It("should be able to clone metrics SLOs", func() { + ref, _, err := metricsStore.Clone( + context.TODO(), + &slov1.SLOData{ + Id: uuid.New().String(), + SLO: util.ProtoClone(mockSLO), + }, + ) + Expect(err).To(Succeed()) + Expect(ref).NotTo(BeNil()) + }) + + It("should be able to clone metrics SLOs across clusters", func() { + refs, _, errs := metricsStore.MultiClusterClone( + context.TODO(), + &slov1.SLOData{ + Id: uuid.New().String(), + SLO: util.ProtoClone(mockSLO), + }, + []*corev1.Reference{ + { + Id: "test", + }, + }, + ) + Expect(errors.Join(errs...)).To(Succeed()) + for _, ref := range refs { + Expect(ref).NotTo(BeNil()) + } + }) + + It("should be able to delete metrics SLOs", func() { + err := metricsStore.Delete( + context.TODO(), + &slov1.SLOData{ + Id: uuid.New().String(), + SLO: util.ProtoClone(mockSLO), + }, + ) + Expect(err).To(Succeed()) + }) + + XIt("should be able to fetch the status of metrics SLOs", func() { + // TODO + _, err := metricsStore.Status( + context.TODO(), + &slov1.SLOData{ + Id: uuid.New().String(), + SLO: util.ProtoClone(mockSLO), + }, + ) + Expect(err).To(Succeed()) + }) + + XIt("should fetch previews of the metrics SLOS", func() { + // TODO + _, err := metricsStore.Preview( + context.TODO(), + &slov1.CreateSLORequest{ + Slo: util.ProtoClone(mockSLO), + }, + ) + Expect(err).To(Succeed()) + }) + }) +}) + +var ( + mockSLO = &slov1.ServiceLevelObjective{ + Name: "test-slo-success", + ClusterId: "agent", + ServiceId: "test-service", + GoodMetricName: "testmetric", + GoodEvents: []*slov1.Event{ + { + Key: "a", + Vals: []string{"a", "b", "c"}, + }, + }, + TotalMetricName: "testmetric", + TotalEvents: []*slov1.Event{}, + Target: &slov1.Target{Value: 99.9}, + SloPeriod: "30d", + BudgetingInterval: durationpb.New(time.Minute * 5), + Datasource: "monitoring", + } +) diff --git a/plugins/slo/pkg/slo/metricgroups/compute.yaml b/pkg/slo/backend/metrics/metricgroups/compute.yaml similarity index 100% rename from plugins/slo/pkg/slo/metricgroups/compute.yaml rename to pkg/slo/backend/metrics/metricgroups/compute.yaml diff --git a/plugins/slo/pkg/slo/metricgroups/go_lang.yaml b/pkg/slo/backend/metrics/metricgroups/go_lang.yaml similarity index 100% rename from plugins/slo/pkg/slo/metricgroups/go_lang.yaml rename to pkg/slo/backend/metrics/metricgroups/go_lang.yaml diff --git a/plugins/slo/pkg/slo/metricgroups/java_lang.yaml b/pkg/slo/backend/metrics/metricgroups/java_lang.yaml similarity index 100% rename from plugins/slo/pkg/slo/metricgroups/java_lang.yaml rename to pkg/slo/backend/metrics/metricgroups/java_lang.yaml diff --git a/plugins/slo/pkg/slo/metricgroups/network.yaml b/pkg/slo/backend/metrics/metricgroups/network.yaml similarity index 100% rename from plugins/slo/pkg/slo/metricgroups/network.yaml rename to pkg/slo/backend/metrics/metricgroups/network.yaml diff --git a/plugins/slo/pkg/slo/metricgroups/other.yaml b/pkg/slo/backend/metrics/metricgroups/other.yaml similarity index 100% rename from plugins/slo/pkg/slo/metricgroups/other.yaml rename to pkg/slo/backend/metrics/metricgroups/other.yaml diff --git a/pkg/slo/backend/metrics/metrics_suite_test.go b/pkg/slo/backend/metrics/metrics_suite_test.go new file mode 100644 index 0000000000..6b56b1be87 --- /dev/null +++ b/pkg/slo/backend/metrics/metrics_suite_test.go @@ -0,0 +1,14 @@ +package metrics_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + _ "github.com/rancher/opni/pkg/test/setup" +) + +func TestMetrics(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Metrics Suite") +} diff --git a/pkg/slo/backend/metrics/prometheus.go b/pkg/slo/backend/metrics/prometheus.go new file mode 100644 index 0000000000..61d3352803 --- /dev/null +++ b/pkg/slo/backend/metrics/prometheus.go @@ -0,0 +1,584 @@ +package metrics + +import ( + "errors" + "fmt" + "text/template" + "time" + + "github.com/prometheus/common/model" + prommodel "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/rulefmt" + "github.com/rancher/opni/pkg/slo/backend" + "gopkg.in/yaml.v3" +) + +var DefaultEvaluationInterval = time.Minute + +const ( + SLOSLI = "slo:sli_error:ratio_rate" + SLOObjective = "slo:objective:ratio" + SLOErrorBudget = "slo:error_budget:ratio" + SLOPeriod = "slo:time_period:duration" + SLOCurrentBurnRate = "slo:current_burn_rate:ratio_rate" + SLOPeriodBurnRate = "slo:period_burn_rate:ratio_rate" + SLOErrorBudgetRemaining = "slo:error_budget_remaining:ratio" + SLOInfo = "opni_slo_info" +) + +func QueryWithLabels(query string, labels backend.IdentificationLabels) string { + return fmt.Sprintf( + "%s{%s}", + query, + labels.ToLabels().ConstructPrometheus(), + ) +} + +type PrometheusSLO struct { + SLI MetricGenerator +} + +type MetricGenerator interface { + Id() string + Expr() string + Rule() (rulefmt.RuleNode, error) +} + +type MWMBSLOGenerator interface { + Windows() []prommodel.Duration + SLI(measurementPeriod prommodel.Duration) MetricGenerator + + Objective() MetricGenerator + ErrorBudget() MetricGenerator + Period() MetricGenerator + Info() MetricGenerator + + CurrentBurnRate() MetricGenerator + PeriodBurnRate() MetricGenerator + ErrorBudgetRemaining() MetricGenerator + + // TODO : unimplemented + // TicketWindow() MetricGenerator + // PageWindow() MetricGenerator + PageAlert() MetricGenerator + TicketAlert() MetricGenerator +} + +type SLOGeneratorOptions struct { + optimized bool +} + +func (s *SLOGeneratorOptions) Apply(opts ...SLOGeneratorOption) { + for _, opt := range opts { + opt(s) + } +} + +func DefaultSLOGeneratorOptions() *SLOGeneratorOptions { + return &SLOGeneratorOptions{ + optimized: true, + } +} + +type SLOGeneratorOption func(*SLOGeneratorOptions) + +func WithOptimization(optimized bool) SLOGeneratorOption { + return func(o *SLOGeneratorOptions) { + o.optimized = optimized + } +} + +func NewSLOGenerator(slo backend.SLO, opts ...SLOGeneratorOption) (*SLOGeneratorImpl, error) { + options := DefaultSLOGeneratorOptions() + options.Apply(opts...) + if err := slo.Validate(); err != nil { + return nil, err + } + dur, err := prommodel.ParseDuration(slo.SloPeriod) + if err != nil { + return nil, err + } + goodQuery := fmt.Sprintf("%s{job=\"%s\", %s}", slo.GoodMetric, slo.Svc, slo.GoodEvents.ConstructPrometheus()) + var totalQuery string + if len(slo.TotalEvents) == 0 { + totalQuery = fmt.Sprintf("%s{job=\"%s\"}", slo.TotalMetric, slo.Svc) + } else { + totalQuery = fmt.Sprintf("%s{job=\"%s\", %s}", slo.TotalMetric, slo.Svc, slo.TotalEvents.ConstructPrometheus()) + } + var windows *MWMBWindows + switch slo.SloPeriod { + case "28d", "30d": //fallback to google SRE production presets + windows = GenerateMWMBWindows( + prommodel.Duration(time.Minute * 5), + ) + default: // use somewhat hacky normalization + windows = GenerateMWMBWindows( + prommodel.Duration(NormalizePeriodToBudgetingInterval(time.Duration(dur))), + ) + } + + return &SLOGeneratorImpl{ + slo: slo, + options: options, + goodQuery: goodQuery, + totalQuery: totalQuery, + period: dur, + windows: windows, + }, nil +} + +type SLOGeneratorImpl struct { + slo backend.SLO + + goodQuery string + totalQuery string + period prommodel.Duration + + windows *MWMBWindows + + options *SLOGeneratorOptions +} + +var _ MWMBSLOGenerator = (*SLOGeneratorImpl)(nil) +var _ RuleGroupBuilder = (*SLOGeneratorImpl)(nil) + +func (s *SLOGeneratorImpl) groupId() string { + return "slo-mwmb" +} + +func (s *SLOGeneratorImpl) AsRuleGroup() (rulefmt.RuleGroup, error) { + rules, err := s.rules() + if err != nil { + return rulefmt.RuleGroup{}, err + } + return rulefmt.RuleGroup{ + Name: s.slo.GetId(), + Interval: prommodel.Duration(DefaultEvaluationInterval), + Rules: rules, + }, nil +} + +func (s *SLOGeneratorImpl) rules() ([]rulefmt.RuleNode, error) { + ruleConstructor := []func() (rulefmt.RuleNode, error){ + s.Objective().Rule, + s.ErrorBudget().Rule, + s.Period().Rule, + s.Info().Rule, + s.CurrentBurnRate().Rule, + s.PeriodBurnRate().Rule, + s.ErrorBudgetRemaining().Rule, + } + + for _, window := range append(s.Windows(), s.period) { + sli := s.SLI(window) + ruleConstructor = append(ruleConstructor, sli.Rule) + } + + rules := []rulefmt.RuleNode{} + var errs []error + for _, constructor := range ruleConstructor { + rule, err := constructor() + if err != nil { + errs = append(errs, err) + continue + } + rules = append(rules, rule) + } + return rules, errors.Join(errs...) +} + +func (s *SLOGeneratorImpl) Windows() []prommodel.Duration { + return s.windows.windowRangeDur() +} + +func (s *SLOGeneratorImpl) SLI(measurementPeriod prommodel.Duration) MetricGenerator { + return NewMWMBSLIGenerator( + SLOSLI, + s.goodQuery, + s.totalQuery, + measurementPeriod, + s.slo.IdLabels, + ) +} + +func normalizeObjective(objective float64) float64 { + return objective / 100 +} + +// ==== constants +func (s *SLOGeneratorImpl) Objective() MetricGenerator { + return NewConstantMetricGenerator(SLOObjective, normalizeObjective(s.slo.Objective), 9, s.slo.IdLabels) +} + +func (s *SLOGeneratorImpl) ErrorBudget() MetricGenerator { + return NewConstantMetricGenerator(SLOErrorBudget, 1-normalizeObjective(s.slo.Objective), 9, s.slo.IdLabels) +} + +func (s *SLOGeneratorImpl) Period() MetricGenerator { + fSeconds := time.Duration(s.period).Seconds() + return NewConstantMetricGenerator(SLOPeriod, fSeconds, 9, s.slo.IdLabels) +} + +func (s *SLOGeneratorImpl) Info() MetricGenerator { + return NewConstantMetricGenerator(SLOInfo, 1, 1, s.slo.IdLabels) +} + +// ==== Dependents on SLI +func (s *SLOGeneratorImpl) CurrentBurnRate() MetricGenerator { + windows := s.Windows() + return NewBurnRateGenerator( + SLOCurrentBurnRate, + s.slo.IdLabels, + s.SLI(windows[0]), + s.ErrorBudget(), + s.options, + ) +} + +func (s *SLOGeneratorImpl) PeriodBurnRate() MetricGenerator { + return NewBurnRateGenerator( + SLOPeriodBurnRate, + s.slo.IdLabels, + s.SLI(s.period), + s.ErrorBudget(), + s.options, + ) +} + +func (s *SLOGeneratorImpl) ErrorBudgetRemaining() MetricGenerator { + return NewErrorBudgetRemainingGenerator( + SLOErrorBudgetRemaining, + s.slo.IdLabels, + s.PeriodBurnRate(), + s.options, + ) +} + +func (s *SLOGeneratorImpl) PageAlert() MetricGenerator { + panic("not implemented") // TODO: Implement +} + +func (s *SLOGeneratorImpl) TicketAlert() MetricGenerator { + panic("not implemented") // TODO: Implement +} + +var _ MWMBSLOGenerator = (*SLOGeneratorImpl)(nil) + +type ConstantMetricGenerator struct { + name string + value float64 + precision int + idLabels backend.IdentificationLabels +} + +var _ MetricGenerator = (*ConstantMetricGenerator)(nil) + +func NewConstantMetricGenerator(id string, value float64, precision int, labels map[string]string) *ConstantMetricGenerator { + return &ConstantMetricGenerator{ + name: id, + value: value, + precision: precision, + idLabels: labels, + } +} + +func (v *ConstantMetricGenerator) Id() string { + return v.name +} + +func (v *ConstantMetricGenerator) Expr() string { + precision := fmt.Sprintf("vector(%%.%df)", v.precision) + return fmt.Sprintf(precision, v.value) +} + +func (v *ConstantMetricGenerator) Rule() (rulefmt.RuleNode, error) { + return rulefmt.RuleNode{ + Record: yaml.Node{ + Kind: yaml.ScalarNode, + Value: v.name, + }, + Expr: yaml.Node{ + Kind: yaml.ScalarNode, + Value: v.Expr(), + }, + Labels: v.idLabels, + }, nil +} + +type SumRateGenerator struct { + goodExpr string + window prommodel.Duration +} + +var _ MetricGenerator = (*SumRateGenerator)(nil) + +func NewSumRateGenerator( + goodExpr string, + window prommodel.Duration, +) *SumRateGenerator { + return &SumRateGenerator{ + goodExpr: goodExpr, + window: window, + } +} + +func (s *SumRateGenerator) Id() string { + return "" +} + +func (s *SumRateGenerator) Expr() string { + return fmt.Sprintf("sum(rate(%s[%s]))", s.goodExpr, s.window.String()) +} + +func (s *SumRateGenerator) Rule() (rulefmt.RuleNode, error) { + return rulefmt.RuleNode{}, fmt.Errorf("partial rules cannot be converted to rulefmt") +} + +func NewMWMBSLIGenerator( + name, goodExpr, totalExpr string, + window prommodel.Duration, + idLabels backend.IdentificationLabels, +) *MWMBSLIGenerator { + goodQuery := NewSumRateGenerator(goodExpr, window) + totalQuery := NewSumRateGenerator(totalExpr, window) + return &MWMBSLIGenerator{ + name: name, + window: window, + goodQuery: goodQuery, + totalQuery: totalQuery, + idLabels: idLabels, + } +} + +type MWMBSLIGenerator struct { + name string + window prommodel.Duration + + goodQuery MetricGenerator + totalQuery MetricGenerator + + idLabels backend.IdentificationLabels +} + +var _ MetricGenerator = (*MWMBSLIGenerator)(nil) + +func (s *MWMBSLIGenerator) Id() string { + return s.name + s.window.String() +} + +func (s *MWMBSLIGenerator) Expr() string { + return fmt.Sprintf("1 - ((%s) / (%s))", s.goodQuery.Expr(), s.totalQuery.Expr()) +} + +func (s *MWMBSLIGenerator) Rule() (rulefmt.RuleNode, error) { + return rulefmt.RuleNode{ + Record: yaml.Node{ + Kind: yaml.ScalarNode, + Value: s.Id(), + }, + Expr: yaml.Node{ + Kind: yaml.ScalarNode, + Value: s.Expr(), + }, + Labels: s.idLabels, + }, nil +} + +type BurnRateGenerator struct { + name string + idLabels backend.IdentificationLabels + + sli MetricGenerator + errorBudget MetricGenerator + + options *SLOGeneratorOptions +} + +var _ MetricGenerator = (*BurnRateGenerator)(nil) + +func (b *BurnRateGenerator) Id() string { + return b.name +} + +func NewBurnRateGenerator( + id string, + idLabels backend.IdentificationLabels, + sli MetricGenerator, + errorBudget MetricGenerator, + options *SLOGeneratorOptions, +) *BurnRateGenerator { + return &BurnRateGenerator{ + name: id, + idLabels: idLabels, + sli: sli, + errorBudget: errorBudget, + options: options, + } +} + +func (b *BurnRateGenerator) Expr() string { + var numerator, denominator string + if b.options.optimized { + numerator = QueryWithLabels(b.sli.Id(), b.idLabels) + denominator = QueryWithLabels(b.errorBudget.Id(), b.idLabels) + return fmt.Sprintf( + "%s / on(%s) group_left %s", + numerator, + b.idLabels.JoinOnPrometheus(), + denominator, + ) + } + numerator = b.sli.Expr() + denominator = b.errorBudget.Expr() + return fmt.Sprintf( + "(%s) / (%s)", + numerator, + denominator, + ) +} + +func (b *BurnRateGenerator) Rule() (rulefmt.RuleNode, error) { + return rulefmt.RuleNode{ + Record: yaml.Node{ + Kind: yaml.ScalarNode, + Value: b.name, + }, + Expr: yaml.Node{ + Kind: yaml.ScalarNode, + Value: b.Expr(), + }, + Labels: b.idLabels, + }, nil +} + +type ErrorBudgetRemainingGenerator struct { + name string + idLabels backend.IdentificationLabels + periodBurnRate MetricGenerator + options *SLOGeneratorOptions +} + +var _ MetricGenerator = (*ErrorBudgetRemainingGenerator)(nil) + +func NewErrorBudgetRemainingGenerator( + id string, + idLabels backend.IdentificationLabels, + periodBurnRate MetricGenerator, + options *SLOGeneratorOptions, +) *ErrorBudgetRemainingGenerator { + return &ErrorBudgetRemainingGenerator{ + name: id, + idLabels: idLabels, + periodBurnRate: periodBurnRate, + options: options, + } +} + +func (e *ErrorBudgetRemainingGenerator) Id() string { + return e.name +} + +func (e *ErrorBudgetRemainingGenerator) Expr() string { + var subExpr string + if e.options.optimized { + subExpr = QueryWithLabels(e.periodBurnRate.Id(), e.idLabels) + } else { + subExpr = e.periodBurnRate.Expr() + } + return fmt.Sprintf("1 - (%s)", subExpr) +} + +func (e *ErrorBudgetRemainingGenerator) Rule() (rulefmt.RuleNode, error) { + return rulefmt.RuleNode{ + Record: yaml.Node{ + Kind: yaml.ScalarNode, + Value: e.name, + }, + Expr: yaml.Node{ + Kind: yaml.ScalarNode, + Value: e.Expr(), + }, + Labels: e.idLabels, + }, nil +} + +var mwmbAlertTplBool = template.Must(template.New("mwmbAlertTpl").Option("missingkey=error").Parse(` +(max({{ .QuickShortMetric }}{{ .MetricFilter}} > bool ({{ .QuickShortBurnFactor }} * {{ .ErrorBudgetRatio }})) +without ({{ .WindowLabel }}) and +max({{ .QuickLongMetric }}{{ .MetricFilter}} > bool ({{ .QuickLongBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }})) + +or + +(max({{ .SlowShortMetric }}{{ .MetricFilter }} > bool ({{ .SlowShortBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }}) +and max({{ .SlowQuickMetric }}{{ .MetricFilter }} > bool ({{ .SlowQuickBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }}))`)) + +// TODO : add a description of how these alerts are structured +type MWMBAlertGenerator struct { + name string + quickWindow Window + slowWindow Window + idLabels backend.IdentificationLabels + options *SLOGeneratorOptions + errorRate func(model.Duration) MetricGenerator +} + +var _ MetricGenerator = (*MWMBAlertGenerator)(nil) + +func NewMWMBAlertGenerator( + name string, + idLabels backend.IdentificationLabels, + quickWindow, slowWindow Window, + errorRate func(model.Duration) MetricGenerator, + options *SLOGeneratorOptions, +) *MWMBAlertGenerator { + return &MWMBAlertGenerator{ + name: name, + idLabels: idLabels, + quickWindow: quickWindow, + slowWindow: slowWindow, + errorRate: errorRate, + options: options, + } +} + +func (m *MWMBAlertGenerator) Id() string { + return m.name +} + +func (m *MWMBAlertGenerator) Expr() string { + construct := func(dur prommodel.Duration) string { + return m.errorRate(dur).Expr() + } + if m.options.optimized { + construct = func(dur prommodel.Duration) string { + return fmt.Sprintf("%s{%s}", m.errorRate(dur).Id(), m.idLabels.ToLabels().ConstructPrometheus()) + } + + } + shortWindowQuickExpr := fmt.Sprintf("(%s) > (%.9f * %.9f)", construct(m.quickWindow.ShortWindow), m.quickWindow.GetShortBurnRateFactor(), m.quickWindow.ErrorBudgetPercent) + shortWindowLongExpr := fmt.Sprintf("(%s) > (%.9f * %.9f)", construct(m.quickWindow.LongWindow), m.quickWindow.GetLongBurnRateFactor(), m.quickWindow.ErrorBudgetPercent) + longWindowQuickExpr := fmt.Sprintf("(%s) > (%.9f * %.9f)", construct(m.slowWindow.ShortWindow), m.slowWindow.GetShortBurnRateFactor(), m.slowWindow.ErrorBudgetPercent) + longWindowLongExpr := fmt.Sprintf("(%s) > (%.9f * %.9f)", construct(m.slowWindow.LongWindow), m.slowWindow.GetLongBurnRateFactor(), m.slowWindow.ErrorBudgetPercent) + + return fmt.Sprintf( + "(max((%s)) and max((%s))) or (max(%s) and max(%s))", + shortWindowQuickExpr, + shortWindowLongExpr, + longWindowQuickExpr, + longWindowLongExpr, + ) +} + +func (m *MWMBAlertGenerator) Rule() (rulefmt.RuleNode, error) { + return rulefmt.RuleNode{ + Alert: yaml.Node{ + Kind: yaml.ScalarNode, + Value: m.Id(), + }, + Expr: yaml.Node{ + Kind: yaml.ScalarNode, + Value: m.Expr(), + }, + Labels: m.idLabels, + }, nil +} diff --git a/pkg/slo/backend/metrics/prometheus_test.go b/pkg/slo/backend/metrics/prometheus_test.go new file mode 100644 index 0000000000..d60e8251d0 --- /dev/null +++ b/pkg/slo/backend/metrics/prometheus_test.go @@ -0,0 +1,990 @@ +package metrics_test + +import ( + "fmt" + "time" + + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + prommodel "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/rulefmt" + promql "github.com/prometheus/prometheus/promql/parser" + "github.com/rancher/opni/pkg/slo/backend" + "github.com/rancher/opni/pkg/slo/backend/metrics" + "github.com/rancher/opni/pkg/util" + "github.com/samber/lo" +) + +var _ = Describe("Prometheus SLO", Label("unit"), func() { + When("We use metric generators", func() { + Specify("constant generators should produce valid promQL", func() { + cnst := metrics.NewConstantMetricGenerator("hello_vector", 0, 9, map[string]string{ + "foo ": "bar", + }) + + Expect(cnst.Id()).To(Equal("hello_vector")) + Expect(cnst.Expr()).To(Equal("vector(0.000000000)")) + + _, err := promql.ParseExpr(cnst.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err := cnst.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(cnst.Id())) + Expect(rule.Expr.Value).To(Equal(cnst.Expr())) + Expect(rule.Labels).To(Equal(map[string]string{ + "foo ": "bar", + })) + }) + + Specify("sum rate generators should produce valid promQL", func() { + sr := metrics.NewSumRateGenerator("hello_vector", prommodel.Duration(time.Minute)) + + // this is a partial generator + Expect(sr.Id()).To(Equal("")) + Expect(sr.Expr()).To(Equal("sum(rate(hello_vector[1m]))")) + _, err := sr.Rule() + Expect(err).To(HaveOccurred()) + + _, err = promql.ParseExpr(sr.Expr()) + Expect(err).NotTo(HaveOccurred()) + + seconds := metrics.NewSumRateGenerator("hello_vector", prommodel.Duration(time.Second)) + Expect(seconds.Id()).To(Equal("")) + Expect(seconds.Expr()).To(Equal("sum(rate(hello_vector[1s]))")) + + _, err = seconds.Rule() + Expect(err).To(HaveOccurred()) + + _, err = promql.ParseExpr(seconds.Expr()) + Expect(err).NotTo(HaveOccurred()) + + hours := metrics.NewSumRateGenerator("hello_vector", prommodel.Duration(time.Hour)) + Expect(hours.Id()).To(Equal("")) + Expect(hours.Expr()).To(Equal("sum(rate(hello_vector[1h]))")) + + _, err = hours.Rule() + Expect(err).To(HaveOccurred()) + + days := metrics.NewSumRateGenerator("hello_vector", prommodel.Duration(24*time.Hour)) + Expect(days.Id()).To(Equal("")) + Expect(days.Expr()).To(Equal("sum(rate(hello_vector[1d]))")) + + _, err = days.Rule() + Expect(err).To(HaveOccurred()) + }) + + Specify("MWMB SLI generators should produce valid promQL", func() { + mwmb := metrics.NewMWMBSLIGenerator( + "sli_vector", + "http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}", + "http_request_duration_seconds_count{job=\"myservice\"}", + prommodel.Duration(time.Minute), + map[string]string{ + "bar": "baz", + }, + ) + + Expect(mwmb.Id()).To(Equal("sli_vector1m")) + Expect(mwmb.Expr()).To(Equal( + "1 - ((sum(rate(http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}[1m]))) / (sum(rate(http_request_duration_seconds_count{job=\"myservice\"}[1m]))))", + )) + + _, err := promql.ParseExpr(mwmb.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err := mwmb.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Labels).To(Equal(map[string]string{ + "bar": "baz", + })) + + _, err = promql.ParseExpr(rule.Record.Value) + Expect(err).NotTo(HaveOccurred()) + + seconds := metrics.NewMWMBSLIGenerator( + "sli_vector", + "http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}", + "http_request_duration_seconds_count{job=\"myservice\"}", + prommodel.Duration(time.Second), + map[string]string{ + "bar": "baz", + }, + ) + + Expect(seconds.Id()).To(Equal("sli_vector1s")) + Expect(seconds.Expr()).To(Equal( + "1 - ((sum(rate(http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}[1s]))) / (sum(rate(http_request_duration_seconds_count{job=\"myservice\"}[1s]))))", + )) + + _, err = promql.ParseExpr(seconds.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = seconds.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Labels).To(Equal(map[string]string{ + "bar": "baz", + })) + + _, err = promql.ParseExpr(rule.Record.Value) + Expect(err).NotTo(HaveOccurred()) + + hours := metrics.NewMWMBSLIGenerator( + "sli_vector", + "http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}", + "http_request_duration_seconds_count{job=\"myservice\"}", + prommodel.Duration(time.Hour*6), + map[string]string{ + "bar": "baz", + }, + ) + + Expect(hours.Id()).To(Equal("sli_vector6h")) + Expect(hours.Expr()).To(Equal( + "1 - ((sum(rate(http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}[6h]))) / (sum(rate(http_request_duration_seconds_count{job=\"myservice\"}[6h]))))", + )) + + _, err = promql.ParseExpr(hours.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = hours.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Labels).To(Equal(map[string]string{ + "bar": "baz", + })) + + _, err = promql.ParseExpr(rule.Record.Value) + Expect(err).NotTo(HaveOccurred()) + + days := metrics.NewMWMBSLIGenerator( + "sli_vector", + "http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}", + "http_request_duration_seconds_count{job=\"myservice\"}", + prommodel.Duration(time.Hour*24*24), + map[string]string{ + "bar": "baz", + }, + ) + + Expect(days.Id()).To(Equal("sli_vector24d")) + Expect(days.Expr()).To(Equal( + "1 - ((sum(rate(http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}[24d]))) / (sum(rate(http_request_duration_seconds_count{job=\"myservice\"}[24d]))))", + )) + + _, err = promql.ParseExpr(days.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = days.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Labels).To(Equal(map[string]string{ + "bar": "baz", + })) + + _, err = promql.ParseExpr(rule.Record.Value) + Expect(err).NotTo(HaveOccurred()) + }) + + Specify("Burn rate generators should generate valid promQL", func() { + idLabels := map[string]string{ + "id": "1", + } + + valueGenerator1 := metrics.NewConstantMetricGenerator( + "test_metric", + 0.5, + 9, + idLabels, + ) + + valueGenerator2 := metrics.NewConstantMetricGenerator( + "test_metric2", + 0.4, + 9, + idLabels, + ) + + opts := metrics.DefaultSLOGeneratorOptions() + + br := metrics.NewBurnRateGenerator( + "burn_rate_current", + idLabels, + valueGenerator1, + valueGenerator2, + opts, + ) + + Expect(br.Id()).To(Equal("burn_rate_current")) + Expect(br.Expr()).To(Equal("test_metric{id=~\"1\"} / on(id) group_left test_metric2{id=~\"1\"}")) + + _, err := promql.ParseExpr(br.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err := br.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Labels).To(Equal(idLabels)) + Expect(rule.Record.Value).To(Equal(br.Id())) + Expect(rule.Expr.Value).To(Equal(br.Expr())) + + idLabels2 := map[string]string{ + "foo": "bar", + "baz": "qux", + } + + br2 := metrics.NewBurnRateGenerator( + "burn_rate_current", + idLabels2, + valueGenerator1, + valueGenerator2, + opts, + ) + + Expect(br2.Id()).To(Equal("burn_rate_current")) + Expect(br2.Expr()).To(Equal("test_metric{baz=~\"qux\",foo=~\"bar\"} / on(baz, foo) group_left test_metric2{baz=~\"qux\",foo=~\"bar\"}")) + + _, err = promql.ParseExpr(br2.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = br2.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Labels).To(Equal(idLabels2)) + Expect(rule.Record.Value).To(Equal(br2.Id())) + Expect(rule.Expr.Value).To(Equal(br2.Expr())) + + By("verifying unpotimized builds return runnable queries") + + opts.Apply( + metrics.WithOptimization(false), + ) + + brUnopt := metrics.NewBurnRateGenerator( + "burn_rate_current", + idLabels2, + valueGenerator1, + valueGenerator2, + opts, + ) + Expect(brUnopt.Id()).To(Equal("burn_rate_current")) + Expect(brUnopt.Expr()).To(Equal("(vector(0.500000000)) / (vector(0.400000000))")) + + _, err = promql.ParseExpr(brUnopt.Expr()) + Expect(err).NotTo(HaveOccurred()) + }) + + Specify("Error budget generators should generate valid promQL", func() { + opts := metrics.DefaultSLOGeneratorOptions() + idLabels := map[string]string{ + "id": "1", + "foo": "bar", + } + periodQuery := metrics.NewConstantMetricGenerator( + "test_metric", + 0.5, + 9, + idLabels, + ) + + errB := metrics.NewErrorBudgetRemainingGenerator( + "hello", + idLabels, + periodQuery, + opts, + ) + Expect(errB.Id()).To(Equal("hello")) + Expect(errB.Expr()).To(Equal("1 - (test_metric{foo=~\"bar\",id=~\"1\"})")) + + _, err := promql.ParseExpr(errB.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err := errB.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Labels).To(Equal(idLabels)) + Expect(rule.Record.Value).To(Equal(errB.Id())) + Expect(rule.Expr.Value).To(Equal(errB.Expr())) + + By("verifying unpotimized builds return runnable queries") + + opts.Apply( + metrics.WithOptimization(false), + ) + + errBUnopt := metrics.NewErrorBudgetRemainingGenerator( + "hello", + idLabels, + periodQuery, + opts, + ) + Expect(errBUnopt.Id()).To(Equal("hello")) + Expect(errBUnopt.Expr()).To(Equal(fmt.Sprintf("1 - (%s)", periodQuery.Expr()))) + + _, err = promql.ParseExpr(errBUnopt.Expr()) + Expect(err).NotTo(HaveOccurred()) + }) + + Specify("MWMB alerts should generate valid promQL", func() { + opts := metrics.DefaultSLOGeneratorOptions() + idLabels := map[string]string{ + "id": "1", + "foo": "bar", + } + errorRate := func(window prommodel.Duration) metrics.MetricGenerator { + return metrics.NewMWMBSLIGenerator( + "error_rate", + "test_metric{foo=~\"bar\",id=~\"1\"}", + "test_metric{foo=~\"bar\",id=~\"1|2\"}", + window, + idLabels, + ) + } + + mwmb := metrics.WindowDefaults(prommodel.Duration(time.Hour * 24 * 30)) + quick, long := mwmb.PageWindows() + pageGen := metrics.NewMWMBAlertGenerator( + "mwmb_alert", + idLabels, + quick, + long, + errorRate, + opts, + ) + + Expect(pageGen.Id()).To(Equal("mwmb_alert")) + GinkgoWriter.Write([]byte(pageGen.Expr())) + // Expect(pageGen.Expr()).Should(Equal("TODO")) // TODO + + rule, err := pageGen.Rule() + Expect(err).NotTo(HaveOccurred()) + + _, err = promql.ParseExpr(rule.Expr.Value) + Expect(err).NotTo(HaveOccurred()) + + Expect(rule.Alert.Value).To(Equal(pageGen.Id())) + Expect(rule.Expr.Value).To(Equal(pageGen.Expr())) + Expect(rule.Labels).To(Equal(idLabels)) + + By("verifying unpotimized MWMB alerts generate valid promQL") + + opts.Apply( + metrics.WithOptimization(false), + ) + pageGenRaw := metrics.NewMWMBAlertGenerator( + "mwmb_alert", + idLabels, + quick, + long, + errorRate, + opts, + ) + + Expect(pageGenRaw.Id()).To(Equal("mwmb_alert")) + // Expect(pageGenRaw.Expr()).Should(Equal("TODO")) // TODO + + rule, err = pageGenRaw.Rule() + Expect(err).NotTo(HaveOccurred()) + + _, err = promql.ParseExpr(rule.Expr.Value) + Expect(err).NotTo(HaveOccurred()) + + Expect(rule.Alert.Value).To(Equal(pageGenRaw.Id())) + Expect(rule.Expr.Value).To(Equal(pageGenRaw.Expr())) + Expect(rule.Labels).To(Equal(idLabels)) + + }) + }) + + When("we use the Prometheus SLO generator", func() { + It("should generate valid MWMB optimized recording rules", func() { + periods := []string{"7m", "7d", "30d"} + for _, period := range periods { + sloUuid := uuid.New().String() + idLabels := backend.IdentificationLabels(map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }) + curSloFilters := idLabels.ToLabels().ConstructPrometheus() + Expect(curSloFilters).To(Equal(fmt.Sprintf("slo_opni_id=~\"%s\",slo_opni_name=~\"test\",slo_opni_service=~\"scrape\"", sloUuid))) + curJoinFilters := idLabels.JoinOnPrometheus() + Expect(curJoinFilters).To(Equal("slo_opni_id, slo_opni_name, slo_opni_service")) + + sloGen, err := metrics.NewSLOGenerator( + backend.SLO{ + SloPeriod: period, + Objective: 99.9, + Svc: "scrape", + GoodMetric: "test_metric", + TotalMetric: "test_metric", + IdLabels: idLabels, + UserLabels: map[string]string{}, + GoodEvents: []backend.LabelPair{ + { + Key: "foo", + Vals: []string{"bar"}, + }, + }, + TotalEvents: []backend.LabelPair{}, + }, + ) + Expect(err).NotTo(HaveOccurred()) + windows := sloGen.Windows() + Expect(windows).NotTo(HaveLen(0)) + Expect(len(windows)).To(BeNumerically(">", 2)) + + for _, window := range windows { + + gen := sloGen.SLI(window) + Expect(gen).NotTo(BeNil()) + Expect(gen.Id()).NotTo(Equal("")) + Expect(gen.Id()).To(HavePrefix(metrics.SLOSLI)) + Expect(gen.Id()).To(HaveSuffix(prommodel.Duration(window).String())) + Expect(gen.Id()).To(Equal(metrics.SLOSLI + prommodel.Duration(window).String())) + expected := fmt.Sprintf("1 - ((sum(rate(test_metric{job=\"scrape\", foo=~\"bar\"}[%s]))) / (sum(rate(test_metric{job=\"scrape\"}[%s]))))", prommodel.Duration(window), prommodel.Duration(window)) + Expect(gen.Expr()).To(Equal(expected)) + _, err = promql.ParseExpr(gen.Expr()) + Expect(err).NotTo(HaveOccurred()) + rule, err := gen.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(gen.Id())) + Expect(rule.Expr.Value).To(Equal(gen.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + } + + By("expecting the constant value rules are correct") + infoVec := sloGen.Info() + Expect(infoVec.Id()).To(Equal(metrics.SLOInfo)) + Expect(infoVec.Expr()).To(Equal("vector(1.0)")) + + _, err = promql.ParseExpr(infoVec.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err := infoVec.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(infoVec.Id())) + Expect(rule.Expr.Value).To(Equal(infoVec.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + periodVec := sloGen.Period() + periodFloat := time.Duration(util.Must(prommodel.ParseDuration(period))).Seconds() + Expect(periodVec.Id()).To(Equal(metrics.SLOPeriod)) + Expect(periodVec.Expr()).To(Equal(fmt.Sprintf("vector(%.9f)", periodFloat))) + + _, err = promql.ParseExpr(periodVec.Expr()) + Expect(err).NotTo(HaveOccurred()) + rule, err = periodVec.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(periodVec.Id())) + Expect(rule.Expr.Value).To(Equal(periodVec.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + objective := sloGen.Objective() + Expect(objective.Id()).To(Equal(metrics.SLOObjective)) + Expect(objective.Expr()).To(Equal("vector(0.999000000)")) + + _, err = promql.ParseExpr(objective.Expr()) + Expect(err).NotTo(HaveOccurred()) + rule, err = objective.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(objective.Id())) + + errorBudget := sloGen.ErrorBudget() + Expect(errorBudget.Id()).To(Equal(metrics.SLOErrorBudget)) + Expect(errorBudget.Expr()).To(Equal("vector(0.001000000)")) + + _, err = promql.ParseExpr(errorBudget.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = errorBudget.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(errorBudget.Id())) + Expect(rule.Expr.Value).To(Equal(errorBudget.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + errorBudgetId := sloGen.ErrorBudget().Id() + By("expecting the optmimized current burn rate rule to reference the correct sli rule") + cbr := sloGen.CurrentBurnRate() + Expect(cbr.Id()).To(Equal(metrics.SLOCurrentBurnRate)) + quickestSLIId := sloGen.SLI(sloGen.Windows()[0]).Id() + expected := fmt.Sprintf( + `%s{%s} / on(%s) group_left %s{%s}`, + quickestSLIId, + curSloFilters, + curJoinFilters, + errorBudgetId, + curSloFilters, + ) + Expect(cbr.Expr()).To(Equal(expected)) + + _, err = promql.ParseExpr(cbr.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = cbr.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(cbr.Id())) + Expect(rule.Expr.Value).To(Equal(cbr.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + By("expecting the optimized period burn rate rule to reference the correct sli rule") + pbr := sloGen.PeriodBurnRate() + + periodSLIID := sloGen.SLI(util.Must(prommodel.ParseDuration(period))).Id() + + expectedP := fmt.Sprintf( + `%s{%s} / on(%s) group_left %s{%s}`, + periodSLIID, + curSloFilters, + curJoinFilters, + errorBudgetId, + curSloFilters, + ) + Expect(pbr.Expr()).To(Equal(expectedP)) + + _, err = promql.ParseExpr(pbr.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = pbr.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(pbr.Id())) + Expect(rule.Expr.Value).To(Equal(pbr.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + By("expecting the optimized error budget remaining rules references the correct metrics") + + errBr := sloGen.ErrorBudgetRemaining() + Expect(errBr.Id()).To(Equal(metrics.SLOErrorBudgetRemaining)) + Expect(errBr.Expr()).To(Equal(fmt.Sprintf("1 - (%s{%s})", pbr.Id(), curSloFilters))) + _, err = promql.ParseExpr(errBr.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = errBr.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(errBr.Id())) + Expect(rule.Expr.Value).To(Equal(errBr.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + } + }) + + It("should generate valid unoptimized MWMB recording rules", func() { + dur := prommodel.Duration((time.Minute*14 + (time.Second * 24)) * 10) + periods := []string{dur.String(), "7d", "30d"} + for _, period := range periods { + sloUuid := uuid.New().String() + idLabels := backend.IdentificationLabels(map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }) + curSloFilters := idLabels.ToLabels().ConstructPrometheus() + Expect(curSloFilters).To(Equal(fmt.Sprintf("slo_opni_id=~\"%s\",slo_opni_name=~\"test\",slo_opni_service=~\"scrape\"", sloUuid))) + curJoinFilters := idLabels.JoinOnPrometheus() + Expect(curJoinFilters).To(Equal("slo_opni_id, slo_opni_name, slo_opni_service")) + + sloGen, err := metrics.NewSLOGenerator( + backend.SLO{ + SloPeriod: period, + Objective: 99.9, + Svc: "scrape", + GoodMetric: "test_metric", + TotalMetric: "test_metric", + IdLabels: idLabels, + UserLabels: map[string]string{}, + GoodEvents: []backend.LabelPair{ + { + Key: "foo", + Vals: []string{"bar"}, + }, + { + Key: "code", + Vals: []string{"200"}, + }, + }, + TotalEvents: []backend.LabelPair{ + { + Key: "code", + Vals: []string{"200", "500", "503"}, + }, + }, + }, + metrics.WithOptimization(false), + ) + Expect(err).NotTo(HaveOccurred()) + windows := sloGen.Windows() + Expect(windows).NotTo(HaveLen(0)) + Expect(len(windows)).To(BeNumerically(">", 2)) + + for _, window := range windows { + gen := sloGen.SLI(window) + Expect(gen).NotTo(BeNil()) + Expect(gen.Id()).NotTo(Equal("")) + Expect(gen.Id()).To(HavePrefix(metrics.SLOSLI)) + Expect(gen.Id()).To(HaveSuffix(prommodel.Duration(window).String())) + Expect(gen.Id()).To(Equal(metrics.SLOSLI + prommodel.Duration(window).String())) + expected := fmt.Sprintf("1 - ((sum(rate(test_metric{job=\"scrape\", foo=~\"bar\",code=~\"200\"}[%s]))) / (sum(rate(test_metric{job=\"scrape\", code=~\"200|500|503\"}[%s]))))", prommodel.Duration(window), prommodel.Duration(window)) + Expect(gen.Expr()).To(Equal(expected)) + _, err := promql.ParseExpr(gen.Expr()) + Expect(err).NotTo(HaveOccurred()) + rule, err := gen.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(gen.Id())) + Expect(rule.Expr.Value).To(Equal(gen.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + } + + By("expecting the constant value rules are correct") + infoVec := sloGen.Info() + Expect(infoVec.Id()).To(Equal(metrics.SLOInfo)) + Expect(infoVec.Expr()).To(Equal("vector(1.0)")) + + _, err = promql.ParseExpr(infoVec.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err := infoVec.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(infoVec.Id())) + Expect(rule.Expr.Value).To(Equal(infoVec.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + periodVec := sloGen.Period() + periodFloat := time.Duration(util.Must(prommodel.ParseDuration(period))).Seconds() + Expect(periodVec.Id()).To(Equal(metrics.SLOPeriod)) + Expect(periodVec.Expr()).To(Equal(fmt.Sprintf("vector(%.9f)", periodFloat))) + + _, err = promql.ParseExpr(periodVec.Expr()) + Expect(err).NotTo(HaveOccurred()) + rule, err = periodVec.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(periodVec.Id())) + Expect(rule.Expr.Value).To(Equal(periodVec.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + objective := sloGen.Objective() + Expect(objective.Id()).To(Equal(metrics.SLOObjective)) + Expect(objective.Expr()).To(Equal("vector(0.999000000)")) + + _, err = promql.ParseExpr(objective.Expr()) + Expect(err).NotTo(HaveOccurred()) + rule, err = objective.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(objective.Id())) + + errorBudget := sloGen.ErrorBudget() + Expect(errorBudget.Id()).To(Equal(metrics.SLOErrorBudget)) + Expect(errorBudget.Expr()).To(Equal("vector(0.001000000)")) + + _, err = promql.ParseExpr(errorBudget.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = errorBudget.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(errorBudget.Id())) + Expect(rule.Expr.Value).To(Equal(errorBudget.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + errorBudgetExpr := sloGen.ErrorBudget().Expr() + By("expecting the unoptmimized current burn rate rule to be valid promQL") + cbr := sloGen.CurrentBurnRate() + Expect(cbr.Id()).To(Equal(metrics.SLOCurrentBurnRate)) + quickestSLIExpr := sloGen.SLI(sloGen.Windows()[0]).Expr() + expected := fmt.Sprintf("(%s) / (%s)", quickestSLIExpr, errorBudgetExpr) + Expect(cbr.Expr()).To(Equal(expected)) + + _, err = promql.ParseExpr(cbr.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = cbr.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(cbr.Id())) + Expect(rule.Expr.Value).To(Equal(cbr.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + By("expecting the unoptimized period burn rate rule to be valid promql") + pbr := sloGen.PeriodBurnRate() + periodSLIExpr := sloGen.SLI(util.Must(prommodel.ParseDuration(period))).Expr() + + expectedP := fmt.Sprintf( + `(%s) / (%s)`, + periodSLIExpr, + errorBudgetExpr, + ) + GinkgoWriter.Write([]byte(expectedP)) + Expect(pbr.Expr()).To(Equal(expectedP)) + + _, err = promql.ParseExpr(pbr.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = pbr.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(pbr.Id())) + Expect(rule.Expr.Value).To(Equal(pbr.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + By("expecting the unoptimized error budget remaining rules references the correct metrics") + + errBr := sloGen.ErrorBudgetRemaining() + Expect(errBr.Id()).To(Equal(metrics.SLOErrorBudgetRemaining)) + Expect(errBr.Expr()).To(Equal(fmt.Sprintf("1 - (%s)", pbr.Expr()))) + + _, err = promql.ParseExpr(errBr.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = errBr.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(errBr.Id())) + Expect(rule.Expr.Value).To(Equal(errBr.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + } + }) + It("should generate valid optimized rule groups", func() { + sloUuid := uuid.New().String() + idLabels := map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + } + curSloFilters := backend.IdentificationLabels(idLabels).ToLabels().ConstructPrometheus() + Expect(curSloFilters).To(Equal(fmt.Sprintf("slo_opni_id=~\"%s\",slo_opni_name=~\"test\",slo_opni_service=~\"scrape\"", sloUuid))) + curJoinFilters := backend.IdentificationLabels(idLabels).JoinOnPrometheus() + Expect(curJoinFilters).To(Equal("slo_opni_id, slo_opni_name, slo_opni_service")) + + period := prommodel.Duration((time.Minute*14 + (time.Second * 24)) * 10) + + sloGen, err := metrics.NewSLOGenerator( + backend.SLO{ + SloPeriod: period.String(), + Objective: 99.9, + Svc: "scrape", + GoodMetric: "test_metric", + TotalMetric: "test_metric", + IdLabels: idLabels, + UserLabels: map[string]string{}, + GoodEvents: []backend.LabelPair{ + { + Key: "foo", + Vals: []string{"bar"}, + }, + { + Key: "code", + Vals: []string{"200"}, + }, + }, + TotalEvents: []backend.LabelPair{ + { + Key: "code", + Vals: []string{"200", "500", "503"}, + }, + }, + }, + metrics.WithOptimization(false), + ) + Expect(err).NotTo(HaveOccurred()) + + rules, err := sloGen.AsRuleGroup() + Expect(err).NotTo(HaveOccurred()) + Expect(rules).NotTo(BeNil()) + Expect(rules.Name).To(ContainSubstring(sloUuid)) + Expect(rules.Rules).NotTo(HaveLen(0)) + + By("verifying the ouput rule groups contain the correct rules") + ruleIds := lo.Map(rules.Rules, func(r rulefmt.RuleNode, _ int) string { + if r.Record.Value == "" { + return r.Alert.Value + } + return r.Record.Value + }) + + generatorIds := []string{ + sloGen.Info().Id(), + sloGen.Objective().Id(), + sloGen.ErrorBudget().Id(), + sloGen.Period().Id(), + sloGen.CurrentBurnRate().Id(), + sloGen.PeriodBurnRate().Id(), + sloGen.ErrorBudgetRemaining().Id(), + } + for _, window := range append(sloGen.Windows(), period) { + generatorIds = append(generatorIds, sloGen.SLI(window).Id()) + } + + Expect(ruleIds).To(ConsistOf(generatorIds)) + + By("verifying the contents are valid promQL and are labelled with the correct ID") + + for _, rule := range rules.Rules { + _, err := promql.ParseExpr(rule.Expr.Value) + Expect(err).NotTo(HaveOccurred()) + + Expect(rule.Labels).To(Equal( + idLabels, + )) + } + }) + + It("should generated valid unoptimized rule groups", func() { + sloUuid := uuid.New().String() + idLabels := map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + } + curSloFilters := backend.IdentificationLabels(idLabels).ToLabels().ConstructPrometheus() + Expect(curSloFilters).To(Equal(fmt.Sprintf("slo_opni_id=~\"%s\",slo_opni_name=~\"test\",slo_opni_service=~\"scrape\"", sloUuid))) + curJoinFilters := backend.IdentificationLabels(idLabels).JoinOnPrometheus() + Expect(curJoinFilters).To(Equal("slo_opni_id, slo_opni_name, slo_opni_service")) + + period := prommodel.Duration((time.Minute*14 + (time.Second * 24)) * 10) + + sloGen, err := metrics.NewSLOGenerator( + backend.SLO{ + SloPeriod: period.String(), + Objective: 99.9, + Svc: "scrape", + GoodMetric: "test_metric", + TotalMetric: "test_metric", + IdLabels: idLabels, + UserLabels: map[string]string{}, + GoodEvents: []backend.LabelPair{ + { + Key: "foo", + Vals: []string{"bar"}, + }, + { + Key: "code", + Vals: []string{"200"}, + }, + }, + TotalEvents: []backend.LabelPair{ + { + Key: "code", + Vals: []string{"200", "500", "503"}, + }, + }, + }, + metrics.WithOptimization(false), + ) + Expect(err).NotTo(HaveOccurred()) + + rules, err := sloGen.AsRuleGroup() + Expect(err).NotTo(HaveOccurred()) + Expect(rules).NotTo(BeNil()) + Expect(rules.Name).To(ContainSubstring(sloUuid)) + Expect(rules.Rules).NotTo(HaveLen(0)) + + By("verifying the ouput rule groups contain the correct rules") + ruleIds := lo.Map(rules.Rules, func(r rulefmt.RuleNode, _ int) string { + if r.Record.Value == "" { + return r.Alert.Value + } + return r.Record.Value + }) + + generatorIds := []string{ + sloGen.Info().Id(), + sloGen.Objective().Id(), + sloGen.ErrorBudget().Id(), + sloGen.Period().Id(), + sloGen.CurrentBurnRate().Id(), + sloGen.PeriodBurnRate().Id(), + sloGen.ErrorBudgetRemaining().Id(), + } + for _, window := range append(sloGen.Windows(), period) { + generatorIds = append(generatorIds, sloGen.SLI(window).Id()) + } + + Expect(ruleIds).To(ConsistOf(generatorIds)) + + By("verifying the contents are valid promQL and are labelled with the correct ID") + + for _, rule := range rules.Rules { + _, err := promql.ParseExpr(rule.Expr.Value) + Expect(err).NotTo(HaveOccurred()) + + Expect(rule.Labels).To(Equal( + idLabels, + )) + } + }) + }) +}) diff --git a/plugins/slo/pkg/slo/servicegroups/kubernetes.yaml b/pkg/slo/backend/metrics/servicegroups/kubernetes.yaml similarity index 87% rename from plugins/slo/pkg/slo/servicegroups/kubernetes.yaml rename to pkg/slo/backend/metrics/servicegroups/kubernetes.yaml index c7960647a6..23775b3884 100644 --- a/plugins/slo/pkg/slo/servicegroups/kubernetes.yaml +++ b/pkg/slo/backend/metrics/servicegroups/kubernetes.yaml @@ -2,20 +2,18 @@ name: "kubernetes metrics" filters: - value : ".*apiserver.*" score : 8 - - value : ".*kube-state-metrics.*" - score : 8 - - value : ".*node-exporter.*" - score : 8 - - value : ".*kube.*" - score : 4 -ignore: - value : ".*kube-proxy.*" score : 10 + - value : ".*kube.*" + score : 4 - value : ".*kubelet.*" score : 8 - - value : ".*coredns.*" +ignore: + - value : ".*kube-state-metrics.*" score : 8 - - value : ".*kube-promethe-operator.*" + - value : ".*node-exporter.*" + score : 8 + - value : ".*coredns.*" score : 8 - value : ".*kube-prometheus-operator.*" score : 8 \ No newline at end of file diff --git a/plugins/slo/pkg/slo/window.go b/pkg/slo/backend/metrics/window.go similarity index 53% rename from plugins/slo/pkg/slo/window.go rename to pkg/slo/backend/metrics/window.go index 53e77132fa..7d038e0f9b 100644 --- a/plugins/slo/pkg/slo/window.go +++ b/pkg/slo/backend/metrics/window.go @@ -1,23 +1,62 @@ -// Reference : https://github.dev/slok/sloth/tree/main/internal/alert/window.go -package slo +package metrics import ( "fmt" "time" + + "slices" + + "github.com/prometheus/common/model" + "github.com/samber/lo" ) -type Windows struct { - SLOPeriod time.Duration +func NormalizePeriodToBudgetingInterval(period time.Duration) (budgetingInterval time.Duration) { + return time.Duration( + int((float64(time.Duration(period).Nanoseconds()) * (float64(5) / float64(43200)))), + ) +} + +type MWMBWindows struct { PageQuick Window PageSlow Window TicketQuick Window TicketSlow Window } +func (w MWMBWindows) windows() []Window { + return []Window{ + w.PageQuick, + w.PageSlow, + w.TicketQuick, + w.TicketSlow, + } +} + +func (w MWMBWindows) WindowRange() []string { + windows := w.windowRangeDur() + return lo.Map(windows, func(w model.Duration, _ int) string { + return w.String() + }) +} + +func (w MWMBWindows) windowRangeDur() []model.Duration { + windows := []model.Duration{} + for _, w := range w.windows() { + windows = append(windows, w.ShortWindow) + windows = append(windows, w.LongWindow) + } + slices.SortFunc(windows, func(i, j model.Duration) int { + return int(int64(i) - int64(j)) + }) + return windows +} + // https://sre.google/workbook/alerting-on-slos/ // // budgeting interval is the shortest interval to monitor in a window -func GenerateGoogleWindows(budgetingInterval time.Duration) *Windows { +// +// budgeting interval should roughly be set to 5/432 of the SLO period +func GenerateMWMBWindows(budgetingInterval model.Duration) *MWMBWindows { pageQuickShort := budgetingInterval pageQuickLong := budgetingInterval * 12 @@ -30,8 +69,7 @@ func GenerateGoogleWindows(budgetingInterval time.Duration) *Windows { ticketSlowShort := ticketQuickShort * 3 ticketSlowLong := ticketQuickLong * 3 - return &Windows{ - SLOPeriod: budgetingInterval, + return &MWMBWindows{ PageQuick: Window{ LongWindow: pageQuickLong, ShortWindow: pageQuickShort, @@ -56,28 +94,31 @@ func GenerateGoogleWindows(budgetingInterval time.Duration) *Windows { } // https://sre.google/workbook/alerting-on-slos/ -func WindowDefaults(period time.Duration) *Windows { - return &Windows{ - SLOPeriod: period, +func WindowDefaults(period model.Duration) *MWMBWindows { + return &MWMBWindows{ PageQuick: Window{ - LongWindow: time.Minute * 60, - ShortWindow: time.Minute * 5, + ShortWindow: model.Duration(time.Minute) * 5, + LongWindow: model.Duration(time.Minute) * 60, ErrorBudgetPercent: 2, + Period: period, }, PageSlow: Window{ - LongWindow: time.Hour * 6, - ShortWindow: time.Minute * 30, + ShortWindow: model.Duration(time.Minute) * 30, + LongWindow: model.Duration(time.Hour) * 6, ErrorBudgetPercent: 5, + Period: period, }, TicketQuick: Window{ - LongWindow: (time.Hour * 24) * 3, - ShortWindow: time.Hour * 6, + LongWindow: (model.Duration(time.Hour) * 24), + ShortWindow: model.Duration(time.Hour) * 2, ErrorBudgetPercent: 10, + Period: period, }, TicketSlow: Window{ - LongWindow: (time.Hour * 24) * 3, - ShortWindow: time.Hour * 6, + LongWindow: (model.Duration(time.Hour) * 24) * 3, + ShortWindow: model.Duration(time.Hour) * 6, ErrorBudgetPercent: 10, + Period: period, }, } } @@ -93,13 +134,17 @@ type Window struct { // ShortWindow is the small window used on the alerting part to stop alerting // during a long window because we consumed a lot of error budget but the problem // is already gone. - ShortWindow time.Duration + ShortWindow model.Duration // LongWindow is the long window used to alert based on the errors happened on that // long window. - LongWindow time.Duration + LongWindow model.Duration + Period model.Duration } func (w Window) Validate() error { + if w.Period == 0 { + return fmt.Errorf("period is required") + } if w.LongWindow == 0 { return fmt.Errorf("long window is required") } @@ -115,10 +160,7 @@ func (w Window) Validate() error { return nil } -func (w Windows) Validate() error { - if w.SLOPeriod == 0 { - return fmt.Errorf("slo period is required") - } +func (w MWMBWindows) Validate() error { err := w.PageQuick.Validate() if err != nil { @@ -143,30 +185,32 @@ func (w Windows) Validate() error { return nil } -// Error budget speeds based on a full time window, however once we have the factor (speed) -// the value can be used with any time window. -func (w Windows) GetSpeedPageQuick() float64 { - return w.GetBurnRateFactor(w.SLOPeriod, float64(w.PageQuick.ErrorBudgetPercent), w.PageQuick.LongWindow) +func (w MWMBWindows) PageWindows() (quickWindow, slowWindow Window) { + return w.PageQuick, w.PageSlow } -func (w Windows) GetSpeedPageSlow() float64 { - return w.GetBurnRateFactor(w.SLOPeriod, float64(w.PageSlow.ErrorBudgetPercent), w.PageSlow.LongWindow) + +func (w MWMBWindows) TicketWindows() (quickWindow, slowWindow Window) { + return w.TicketQuick, w.TicketSlow } -func (w Windows) GetSpeedTicketQuick() float64 { - return w.GetBurnRateFactor(w.SLOPeriod, float64(w.TicketQuick.ErrorBudgetPercent), w.TicketQuick.LongWindow) + +// Error budget speeds based on a full time window, however once we have the factor (speed) +// the value can be used with any time window. +func (w Window) GetLongBurnRateFactor() float64 { + return GetBurnRateFactor(w.Period, w.ErrorBudgetPercent, w.LongWindow) } -func (w Windows) GetSpeedTicketSlow() float64 { - return w.GetBurnRateFactor(w.SLOPeriod, float64(w.TicketSlow.ErrorBudgetPercent), w.TicketSlow.LongWindow) + +func (w Window) GetShortBurnRateFactor() float64 { + return GetBurnRateFactor(w.Period, w.ErrorBudgetPercent, w.ShortWindow) } // getBurnRateFactor calculates the burnRateFactor (speed) needed to consume all the error budget available percent // in a specific time window taking into account the total time window. -func (w Windows) GetBurnRateFactor(totalWindow time.Duration, errorBudgetPercent float64, consumptionWindow time.Duration) float64 { - // First get the total hours required to consume the % of the error budget in the total window. - hoursRequiredConsumption := errorBudgetPercent * totalWindow.Hours() / 100 - +// returns a value between 0 and 100 +func GetBurnRateFactor(totalWindow model.Duration, errorBudgetPercent float64, consumptionWindow model.Duration) float64 { + // First get the total seconds required to consume the % of the error budget in the total window. + secondsRequiredConsumption := (errorBudgetPercent * time.Duration(totalWindow).Seconds()) / 100 // Now calculate how much is the factor required for the hours consumption, in case we would need to use // a different time window (e.g: hours required: 36h, if we want to do it in 6h: would be `x6`). - speed := hoursRequiredConsumption / consumptionWindow.Hours() - + speed := secondsRequiredConsumption / time.Duration(consumptionWindow).Seconds() return speed } diff --git a/pkg/slo/backend/metrics/window_test.go b/pkg/slo/backend/metrics/window_test.go new file mode 100644 index 0000000000..aa1af4efff --- /dev/null +++ b/pkg/slo/backend/metrics/window_test.go @@ -0,0 +1,51 @@ +package metrics_test + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/prometheus/common/model" + "github.com/rancher/opni/pkg/slo/backend/metrics" +) + +var _ = XDescribe("Windows test", Label("unit"), func() { + DescribeTable("invalid Window struct", func(window metrics.Window) { + Expect(window.Validate()).NotTo(Succeed()) + }, + Entry("Empty window", metrics.Window{}), + Entry("missing long window", metrics.Window{ + ShortWindow: model.Duration(time.Minute), + ErrorBudgetPercent: 2, + }), + Entry("missing short window", metrics.Window{ + LongWindow: model.Duration(time.Hour), + ErrorBudgetPercent: 2, + }), + Entry("missing error budget", metrics.Window{ + LongWindow: model.Duration(time.Hour), + ShortWindow: model.Duration(time.Minute), + }), + ) + + DescribeTable("valid windows -- duration strings", func(window *metrics.MWMBWindows, windowRange []string) { + Expect(window.Validate()).To(Succeed()) + Expect(window.WindowRange()).To(Equal(windowRange)) + }, + Entry("default window", metrics.WindowDefaults(model.Duration(time.Hour*24*30)), []string{"5m", "30m", "1h", "2h", "6h", "1d", "3d"}), + Entry("custom window", metrics.GenerateMWMBWindows(model.Duration(time.Minute*5)), []string{"5m", "30m", "1h", "2h", "6h", "1d", "3d"}), + Entry("custom window short", metrics.GenerateMWMBWindows(model.Duration(time.Second*5)), []string{"5s", "30s", "1m", "2m", "6m", "24m", "1h12m"}), + Entry("normalizing period interval", + metrics.GenerateMWMBWindows( + model.Duration(metrics.NormalizePeriodToBudgetingInterval(time.Minute*72*10))), + []string{"5s", "30s", "1m", "2m", "6m", "24m", "1h12m"}, + ), + Entry("custom window very short", metrics.GenerateMWMBWindows(model.Duration(time.Second)), []string{"1s", "6s", "12s", "24s", "1m12s", "4m48s", "14m24s"}), + Entry("normalizing very short interval", + metrics.GenerateMWMBWindows( + model.Duration(metrics.NormalizePeriodToBudgetingInterval((time.Minute*14+(time.Second*24)))*10), + ), + []string{"1s", "6s", "12s", "24s", "1m12s", "4m48s", "14m24s"}, + ), + ) +}) diff --git a/pkg/slo/backend/service.go b/pkg/slo/backend/service.go new file mode 100644 index 0000000000..4a75395aab --- /dev/null +++ b/pkg/slo/backend/service.go @@ -0,0 +1,13 @@ +package backend + +import ( + "context" + + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" +) + +type ServiceBackend interface { + ListServices(ctx context.Context, req *slov1.ListServicesRequest) (*slov1.ServiceList, error) + ListMetrics(ctx context.Context, req *slov1.ListMetricsRequest) (*slov1.MetricGroupList, error) + ListEvents(ctx context.Context, req *slov1.ListEventsRequest) (*slov1.EventList, error) +} diff --git a/pkg/slo/backend/slo.go b/pkg/slo/backend/slo.go new file mode 100644 index 0000000000..bfa46858bd --- /dev/null +++ b/pkg/slo/backend/slo.go @@ -0,0 +1,308 @@ +package backend + +import ( + "context" + "fmt" + "slices" + + "github.com/google/uuid" + corev1 "github.com/rancher/opni/pkg/apis/core/v1" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/samber/lo" +) + +// TODO : tests +type SLO struct { + SloPeriod string + Objective float64 // 0 < x < 100 + Svc Service + GoodMetric Metric + TotalMetric Metric + IdLabels IdentificationLabels + UserLabels map[string]string + GoodEvents LabelPairs + TotalEvents LabelPairs +} + +type SLODatasource interface { + SLOStore + ServiceBackend + Precondition(ctx context.Context, clusterId *corev1.Reference) error +} + +type sLODatasourceImpl struct { + SLOStore + ServiceBackend + precondition func(ctx context.Context, clusterId *corev1.Reference) error +} + +func (s *sLODatasourceImpl) Precondition(ctx context.Context, clusterId *corev1.Reference) error { + return s.precondition(ctx, clusterId) +} + +func NewSLODatasource( + sloStore SLOStore, + serviceBackend ServiceBackend, + precondition func(ctx context.Context, clusterId *corev1.Reference) error, +) SLODatasource { + return &sLODatasourceImpl{ + SLOStore: sloStore, + ServiceBackend: serviceBackend, + precondition: precondition, + } +} + +func (s *SLO) Validate() error { + if s.SloPeriod == "" { + return fmt.Errorf("slo period is required") + } + if s.Objective <= 0 || s.Objective >= 100 { + return fmt.Errorf("slo object must be between 0 and 100 (exclusive)") + } + if s.Svc == "" { + return fmt.Errorf("slo service is required") + } + if s.GoodMetric == "" { + return fmt.Errorf("slo good metric is required") + } + if s.TotalMetric == "" { + return fmt.Errorf("slo total metric is required") + } + if len(s.GoodEvents) == 0 { + return fmt.Errorf("slo good events are required") + } + if err := s.IdLabels.Validate(); err != nil { + return err + } + return nil +} + +func (s *SLO) GetId() string { + return s.IdLabels[SLOUuid] +} + +func (s *SLO) SetId(id string) { + s.IdLabels[SLOUuid] = id +} + +func (s *SLO) GetName() string { + return s.IdLabels[SLOName] +} + +func (s *SLO) SetName(input string) { + s.IdLabels[SLOName] = input +} + +func (s *SLO) GetPeriod() string { + return s.SloPeriod +} + +func (s *SLO) GetObjective() float64 { + return s.Objective +} + +// MatchEventsOnMetric only applies when the good metric & total metric id is the same +func MatchEventsOnMetric(goodEvents, totalEvents []*slov1.Event) (good, total []*slov1.Event) { + goodAggregate := map[string][]string{} + for _, g := range goodEvents { + if g.Key == "" || g.Vals == nil { + continue + } + if _, ok := goodAggregate[g.Key]; !ok { + goodAggregate[g.Key] = []string{} + } + goodAggregate[g.Key] = append(goodAggregate[g.Key], g.Vals...) + } + totalAggregate := map[string][]string{} + for _, t := range totalEvents { + if t.Key == "" || t.Vals == nil { + continue + } + if _, ok := totalAggregate[t.Key]; !ok { + totalAggregate[t.Key] = []string{} + } + totalAggregate[t.Key] = append(totalAggregate[t.Key], t.Vals...) + } + for key, vals := range totalAggregate { + if _, ok := goodAggregate[key]; ok { + // total event filter defined on good and total so reconcile + totalAggregate[key] = LeftJoinSlice(goodAggregate[key], vals) + } else { + // total filter must also apply to good event filter + goodAggregate[key] = vals + } + } + retGoodEvent := lo.MapToSlice(goodAggregate, func(k string, v []string) *slov1.Event { + return &slov1.Event{ + Key: k, + Vals: v, + } + }) + retTotalEvent := lo.MapToSlice(totalAggregate, func(k string, v []string) *slov1.Event { + return &slov1.Event{ + Key: k, + Vals: v, + } + }) + + slices.SortFunc(retTotalEvent, func(a, b *slov1.Event) int { + if a.Key < b.Key { + return -1 + } else if a.Key > b.Key { + return 1 + } + return 0 + }) + + slices.SortFunc(retGoodEvent, func(a, b *slov1.Event) int { + if a.Key < b.Key { + return -1 + } else if a.Key > b.Key { + return 1 + } + return 0 + }) + return retGoodEvent, retTotalEvent +} + +func CreateSLORequestToStruct(c *slov1.CreateSLORequest) *SLO { + if c.Slo.GetGoodMetricName() == c.Slo.GetTotalMetricName() { + c.Slo.GoodEvents, c.Slo.TotalEvents = MatchEventsOnMetric(c.Slo.GoodEvents, c.Slo.TotalEvents) + } + reqSLO := c.Slo + userLabels := reqSLO.GetLabels() + sloLabels := map[string]string{} + for _, label := range userLabels { + sloLabels[label.GetName()] = "true" + } + goodEvents := []LabelPair{} + for _, goodEvent := range reqSLO.GetGoodEvents() { + goodEvents = append(goodEvents, LabelPair{ + Key: goodEvent.GetKey(), + Vals: goodEvent.GetVals(), + }) + } + totalEvents := []LabelPair{} + for _, totalEvent := range reqSLO.GetTotalEvents() { + totalEvents = append(totalEvents, LabelPair{ + Key: totalEvent.GetKey(), + Vals: totalEvent.GetVals(), + }) + } + return NewSLO( + reqSLO.GetName(), + reqSLO.GetSloPeriod(), + reqSLO.GetTarget().GetValue(), + Service(reqSLO.GetServiceId()), + Metric(reqSLO.GetGoodMetricName()), + Metric(reqSLO.GetTotalMetricName()), + sloLabels, + goodEvents, + totalEvents, + ) +} + +func NewSLO( + sloName string, + sloPeriod string, + objective float64, + svc Service, + goodMetric Metric, + totalMetric Metric, + userLabels map[string]string, + goodEvents []LabelPair, + totalEvents []LabelPair, +) *SLO { + newId := uuid.New().String() + ilabels := IdentificationLabels{SLOUuid: newId, SLOName: sloName, SLOService: string(svc)} + + return &SLO{ + Svc: svc, + SloPeriod: sloPeriod, + GoodMetric: goodMetric, + TotalMetric: totalMetric, + UserLabels: userLabels, + GoodEvents: goodEvents, + TotalEvents: totalEvents, + IdLabels: ilabels, + Objective: objective, + } +} + +func SLOFromId( + sloName string, + sloPeriod string, + objective float64, + svc Service, + goodMetric Metric, + totalMetric Metric, + userLabels map[string]string, + goodEvents []LabelPair, + totalEvents []LabelPair, + id string, +) *SLO { + ilabels := IdentificationLabels{SLOUuid: id, SLOName: sloName, SLOService: string(svc)} + + return &SLO{ + Svc: svc, + GoodMetric: goodMetric, + TotalMetric: totalMetric, + SloPeriod: sloPeriod, + UserLabels: userLabels, + GoodEvents: goodEvents, + TotalEvents: totalEvents, + IdLabels: ilabels, + Objective: objective, + } +} + +func SLODataToStruct(s *slov1.SLOData) *SLO { + reqSLO := s.SLO + if reqSLO.GetGoodMetricName() == reqSLO.GetTotalMetricName() { + reqSLO.GoodEvents, reqSLO.TotalEvents = MatchEventsOnMetric(reqSLO.GoodEvents, reqSLO.TotalEvents) + } + userLabels := reqSLO.GetLabels() + sloLabels := map[string]string{} + for _, label := range userLabels { + sloLabels[label.GetName()] = "true" + } + goodEvents := []LabelPair{} + for _, goodEvent := range reqSLO.GetGoodEvents() { + goodEvents = append(goodEvents, LabelPair{ + Key: goodEvent.GetKey(), + Vals: goodEvent.GetVals(), + }) + } + totalEvents := []LabelPair{} + for _, totalEvent := range reqSLO.GetTotalEvents() { + totalEvents = append(totalEvents, LabelPair{ + Key: totalEvent.GetKey(), + Vals: totalEvent.GetVals(), + }) + } + if s.Id == "" { + return NewSLO( + reqSLO.GetName(), + reqSLO.GetSloPeriod(), + reqSLO.GetTarget().GetValue(), + Service(reqSLO.GetServiceId()), + Metric(reqSLO.GetGoodMetricName()), + Metric(reqSLO.GetTotalMetricName()), + sloLabels, + goodEvents, + totalEvents, + ) + } + return SLOFromId( + reqSLO.GetName(), + reqSLO.GetSloPeriod(), + reqSLO.GetTarget().GetValue(), + Service(reqSLO.GetServiceId()), + Metric(reqSLO.GetGoodMetricName()), + Metric(reqSLO.GetTotalMetricName()), + sloLabels, + goodEvents, + totalEvents, + s.Id, + ) +} diff --git a/pkg/slo/backend/slo_test.go b/pkg/slo/backend/slo_test.go new file mode 100644 index 0000000000..d02f65a55f --- /dev/null +++ b/pkg/slo/backend/slo_test.go @@ -0,0 +1,3 @@ +package backend_test + +// TODO : diff --git a/pkg/slo/backend/store.go b/pkg/slo/backend/store.go new file mode 100644 index 0000000000..c6f48b1d08 --- /dev/null +++ b/pkg/slo/backend/store.go @@ -0,0 +1,24 @@ +package backend + +import ( + "context" + + corev1 "github.com/rancher/opni/pkg/apis/core/v1" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" +) + +type SLOStore interface { + Create(ctx context.Context, req *slov1.CreateSLORequest) (*corev1.Reference, error) + Update(ctx context.Context, incoming, existing *slov1.SLOData) (*slov1.SLOData, error) + Delete(ctx context.Context, existing *slov1.SLOData) error + + Clone(ctx context.Context, clone *slov1.SLOData) (*corev1.Reference, *slov1.SLOData, error) + MultiClusterClone( + ctx context.Context, + slo *slov1.SLOData, + toClusters []*corev1.Reference, + ) ([]*corev1.Reference, []*slov1.SLOData, []error) + + Status(ctx context.Context, existing *slov1.SLOData) (*slov1.SLOStatus, error) + Preview(ctx context.Context, s *slov1.CreateSLORequest) (*slov1.SLOPreviewResponse, error) +} diff --git a/pkg/slo/backend/util.go b/pkg/slo/backend/util.go new file mode 100644 index 0000000000..34d9d8b556 --- /dev/null +++ b/pkg/slo/backend/util.go @@ -0,0 +1,96 @@ +package backend + +import ( + "fmt" + "slices" + "sort" + "strings" + + "github.com/samber/lo" +) + +const ( + // id labels + SLOUuid = "slo_opni_id" + SLOService = "slo_opni_service" + SLOName = "slo_opni_name" +) + +type Service string +type Metric string + +type LabelPairs []LabelPair +type LabelPair struct { + Key string + Vals []string +} + +type IdentificationLabels map[string]string + +func (i IdentificationLabels) Validate() error { + if _, ok := i[SLOUuid]; !ok { + return fmt.Errorf("slo uuid label '%s' is required", SLOUuid) + } + if _, ok := i[SLOService]; !ok { + return fmt.Errorf("slo service label '%s' is required", SLOService) + } + if _, ok := i[SLOName]; !ok { + return fmt.Errorf("slo name label '%s' is required", SLOName) + } + return nil +} + +func (i IdentificationLabels) ToLabels() LabelPairs { + lbs := lo.MapToSlice(i, func(k string, v string) LabelPair { + return LabelPair{ + Key: k, + Vals: []string{v}, + } + }) + + slices.SortFunc(lbs, func(a, b LabelPair) int { + if a.Key < b.Key { + return -1 + } else if a.Key > b.Key { + return 1 + } + return 0 + }) + return lbs +} + +func (i IdentificationLabels) JoinOnPrometheus() string { + keys := lo.Keys(i) + sort.Strings(keys) + return strings.Join(keys, ", ") +} + +func (l LabelPairs) ConstructPrometheus() string { // kinda hacky & technically unoptimized but works + if len(l) == 0 { + return "" + } + filterStrings := []string{} + for _, labelPair := range l { + if labelPair.Key == "" || len(labelPair.Vals) == 0 { + continue + } + filterString := fmt.Sprintf("%s=~\"%s\"", labelPair.Key, strings.Join(labelPair.Vals, "|")) + filterStrings = append(filterStrings, filterString) + } + return strings.Join(filterStrings, ",") +} + +func LeftJoinSlice[T comparable](arr1, arr2 []T) []T { + result := make([]T, len(arr1)) + cache := map[T]struct{}{} + for i, v := range arr1 { + cache[v] = struct{}{} + result[i] = v + } + for _, v := range arr2 { + if _, ok := cache[v]; !ok { + result = append(result, v) + } + } + return result +} diff --git a/pkg/slo/backend/util_test.go b/pkg/slo/backend/util_test.go new file mode 100644 index 0000000000..57fd51ff03 --- /dev/null +++ b/pkg/slo/backend/util_test.go @@ -0,0 +1,60 @@ +package backend_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/rancher/opni/pkg/slo/backend" +) + +var _ = Describe("SLO util", Label("unit"), func() { + When("we use id labels util", func() { + Specify("it should convert to prometheus join on filters", func() { + i := backend.IdentificationLabels(map[string]string{ + "foo": "bar", + }) + Expect(i.JoinOnPrometheus()).To(Equal("foo")) + + i2 := backend.IdentificationLabels(map[string]string{ + "foo": "bar", + "baz": "qux", + }) + + Expect(i2.JoinOnPrometheus()).To(Equal("baz, foo")) + }) + + Specify("it should convert to SLO label pairs", func() { + i := backend.IdentificationLabels(map[string]string{ + "foo": "bar", + "baz": "qux", + }) + + Expect(i.ToLabels()).To(Equal(backend.LabelPairs{ + { + Key: "baz", + Vals: []string{"qux"}, + }, + { + Key: "foo", + Vals: []string{"bar"}, + }, + })) + }) + }) + + When("we use label pairs util", func() { + Specify("it should construct prometheus join on filters", func() { + lb := backend.LabelPairs{ + { + Key: "foo", + Vals: []string{"bar"}, + }, + { + Key: "baz", + Vals: []string{"qux", "qux2"}, + }, + } + filter := lb.ConstructPrometheus() + Expect(filter).To(Equal("foo=~\"bar\",baz=~\"qux|qux2\"")) + }) + }) +}) diff --git a/pkg/slo/query/query_test.go b/pkg/slo/query/query_test.go index 170f9c48f2..d1b10dc336 100644 --- a/pkg/slo/query/query_test.go +++ b/pkg/slo/query/query_test.go @@ -3,8 +3,8 @@ package query_test import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + api "github.com/rancher/opni/pkg/apis/slo/v1" "github.com/rancher/opni/pkg/slo/query" - api "github.com/rancher/opni/plugins/slo/apis/slo" ) func constructionShouldSucceed(q *query.SLOQueryResult, err error) { diff --git a/pkg/test/mock/cortexadmin/cortexadmin.go b/pkg/test/mock/cortexadmin/cortexadmin.go new file mode 100644 index 0000000000..3457d5ecbb --- /dev/null +++ b/pkg/test/mock/cortexadmin/cortexadmin.go @@ -0,0 +1,633 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: plugins/metrics/apis/cortexadmin/cortexadmin_grpc.pb.go + +// Package mock_cortexadmin is a generated GoMock package. +package mock_cortexadmin + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + cortexadmin "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" + grpc "google.golang.org/grpc" + emptypb "google.golang.org/protobuf/types/known/emptypb" +) + +// MockCortexAdminClient is a mock of CortexAdminClient interface. +type MockCortexAdminClient struct { + ctrl *gomock.Controller + recorder *MockCortexAdminClientMockRecorder +} + +// MockCortexAdminClientMockRecorder is the mock recorder for MockCortexAdminClient. +type MockCortexAdminClientMockRecorder struct { + mock *MockCortexAdminClient +} + +// NewMockCortexAdminClient creates a new mock instance. +func NewMockCortexAdminClient(ctrl *gomock.Controller) *MockCortexAdminClient { + mock := &MockCortexAdminClient{ctrl: ctrl} + mock.recorder = &MockCortexAdminClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCortexAdminClient) EXPECT() *MockCortexAdminClientMockRecorder { + return m.recorder +} + +// AllUserStats mocks base method. +func (m *MockCortexAdminClient) AllUserStats(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*cortexadmin.UserIDStatsList, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "AllUserStats", varargs...) + ret0, _ := ret[0].(*cortexadmin.UserIDStatsList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AllUserStats indicates an expected call of AllUserStats. +func (mr *MockCortexAdminClientMockRecorder) AllUserStats(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllUserStats", reflect.TypeOf((*MockCortexAdminClient)(nil).AllUserStats), varargs...) +} + +// DeleteRule mocks base method. +func (m *MockCortexAdminClient) DeleteRule(ctx context.Context, in *cortexadmin.DeleteRuleRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DeleteRule", varargs...) + ret0, _ := ret[0].(*emptypb.Empty) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteRule indicates an expected call of DeleteRule. +func (mr *MockCortexAdminClientMockRecorder) DeleteRule(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRule", reflect.TypeOf((*MockCortexAdminClient)(nil).DeleteRule), varargs...) +} + +// ExtractRawSeries mocks base method. +func (m *MockCortexAdminClient) ExtractRawSeries(ctx context.Context, in *cortexadmin.MatcherRequest, opts ...grpc.CallOption) (*cortexadmin.QueryResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ExtractRawSeries", varargs...) + ret0, _ := ret[0].(*cortexadmin.QueryResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ExtractRawSeries indicates an expected call of ExtractRawSeries. +func (mr *MockCortexAdminClientMockRecorder) ExtractRawSeries(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExtractRawSeries", reflect.TypeOf((*MockCortexAdminClient)(nil).ExtractRawSeries), varargs...) +} + +// FlushBlocks mocks base method. +func (m *MockCortexAdminClient) FlushBlocks(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*emptypb.Empty, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "FlushBlocks", varargs...) + ret0, _ := ret[0].(*emptypb.Empty) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FlushBlocks indicates an expected call of FlushBlocks. +func (mr *MockCortexAdminClientMockRecorder) FlushBlocks(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FlushBlocks", reflect.TypeOf((*MockCortexAdminClient)(nil).FlushBlocks), varargs...) +} + +// GetCortexConfig mocks base method. +func (m *MockCortexAdminClient) GetCortexConfig(ctx context.Context, in *cortexadmin.ConfigRequest, opts ...grpc.CallOption) (*cortexadmin.ConfigResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetCortexConfig", varargs...) + ret0, _ := ret[0].(*cortexadmin.ConfigResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCortexConfig indicates an expected call of GetCortexConfig. +func (mr *MockCortexAdminClientMockRecorder) GetCortexConfig(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCortexConfig", reflect.TypeOf((*MockCortexAdminClient)(nil).GetCortexConfig), varargs...) +} + +// GetCortexStatus mocks base method. +func (m *MockCortexAdminClient) GetCortexStatus(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*cortexadmin.CortexStatus, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetCortexStatus", varargs...) + ret0, _ := ret[0].(*cortexadmin.CortexStatus) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCortexStatus indicates an expected call of GetCortexStatus. +func (mr *MockCortexAdminClientMockRecorder) GetCortexStatus(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCortexStatus", reflect.TypeOf((*MockCortexAdminClient)(nil).GetCortexStatus), varargs...) +} + +// GetMetricLabelSets mocks base method. +func (m *MockCortexAdminClient) GetMetricLabelSets(ctx context.Context, in *cortexadmin.LabelRequest, opts ...grpc.CallOption) (*cortexadmin.MetricLabels, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetMetricLabelSets", varargs...) + ret0, _ := ret[0].(*cortexadmin.MetricLabels) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetMetricLabelSets indicates an expected call of GetMetricLabelSets. +func (mr *MockCortexAdminClientMockRecorder) GetMetricLabelSets(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMetricLabelSets", reflect.TypeOf((*MockCortexAdminClient)(nil).GetMetricLabelSets), varargs...) +} + +// GetMetricMetadata mocks base method. +func (m *MockCortexAdminClient) GetMetricMetadata(ctx context.Context, in *cortexadmin.MetricMetadataRequest, opts ...grpc.CallOption) (*cortexadmin.MetricMetadata, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetMetricMetadata", varargs...) + ret0, _ := ret[0].(*cortexadmin.MetricMetadata) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetMetricMetadata indicates an expected call of GetMetricMetadata. +func (mr *MockCortexAdminClientMockRecorder) GetMetricMetadata(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMetricMetadata", reflect.TypeOf((*MockCortexAdminClient)(nil).GetMetricMetadata), varargs...) +} + +// GetRule mocks base method. +func (m *MockCortexAdminClient) GetRule(ctx context.Context, in *cortexadmin.GetRuleRequest, opts ...grpc.CallOption) (*cortexadmin.QueryResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetRule", varargs...) + ret0, _ := ret[0].(*cortexadmin.QueryResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetRule indicates an expected call of GetRule. +func (mr *MockCortexAdminClientMockRecorder) GetRule(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRule", reflect.TypeOf((*MockCortexAdminClient)(nil).GetRule), varargs...) +} + +// GetSeriesMetrics mocks base method. +func (m *MockCortexAdminClient) GetSeriesMetrics(ctx context.Context, in *cortexadmin.SeriesRequest, opts ...grpc.CallOption) (*cortexadmin.SeriesInfoList, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetSeriesMetrics", varargs...) + ret0, _ := ret[0].(*cortexadmin.SeriesInfoList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSeriesMetrics indicates an expected call of GetSeriesMetrics. +func (mr *MockCortexAdminClientMockRecorder) GetSeriesMetrics(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSeriesMetrics", reflect.TypeOf((*MockCortexAdminClient)(nil).GetSeriesMetrics), varargs...) +} + +// ListRules mocks base method. +func (m *MockCortexAdminClient) ListRules(ctx context.Context, in *cortexadmin.ListRulesRequest, opts ...grpc.CallOption) (*cortexadmin.ListRulesResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListRules", varargs...) + ret0, _ := ret[0].(*cortexadmin.ListRulesResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRules indicates an expected call of ListRules. +func (mr *MockCortexAdminClientMockRecorder) ListRules(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRules", reflect.TypeOf((*MockCortexAdminClient)(nil).ListRules), varargs...) +} + +// LoadRules mocks base method. +func (m *MockCortexAdminClient) LoadRules(ctx context.Context, in *cortexadmin.LoadRuleRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "LoadRules", varargs...) + ret0, _ := ret[0].(*emptypb.Empty) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// LoadRules indicates an expected call of LoadRules. +func (mr *MockCortexAdminClientMockRecorder) LoadRules(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadRules", reflect.TypeOf((*MockCortexAdminClient)(nil).LoadRules), varargs...) +} + +// Query mocks base method. +func (m *MockCortexAdminClient) Query(ctx context.Context, in *cortexadmin.QueryRequest, opts ...grpc.CallOption) (*cortexadmin.QueryResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Query", varargs...) + ret0, _ := ret[0].(*cortexadmin.QueryResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Query indicates an expected call of Query. +func (mr *MockCortexAdminClientMockRecorder) Query(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Query", reflect.TypeOf((*MockCortexAdminClient)(nil).Query), varargs...) +} + +// QueryRange mocks base method. +func (m *MockCortexAdminClient) QueryRange(ctx context.Context, in *cortexadmin.QueryRangeRequest, opts ...grpc.CallOption) (*cortexadmin.QueryResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "QueryRange", varargs...) + ret0, _ := ret[0].(*cortexadmin.QueryResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// QueryRange indicates an expected call of QueryRange. +func (mr *MockCortexAdminClientMockRecorder) QueryRange(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueryRange", reflect.TypeOf((*MockCortexAdminClient)(nil).QueryRange), varargs...) +} + +// WriteMetrics mocks base method. +func (m *MockCortexAdminClient) WriteMetrics(ctx context.Context, in *cortexadmin.WriteRequest, opts ...grpc.CallOption) (*cortexadmin.WriteResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "WriteMetrics", varargs...) + ret0, _ := ret[0].(*cortexadmin.WriteResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// WriteMetrics indicates an expected call of WriteMetrics. +func (mr *MockCortexAdminClientMockRecorder) WriteMetrics(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteMetrics", reflect.TypeOf((*MockCortexAdminClient)(nil).WriteMetrics), varargs...) +} + +// MockCortexAdminServer is a mock of CortexAdminServer interface. +type MockCortexAdminServer struct { + ctrl *gomock.Controller + recorder *MockCortexAdminServerMockRecorder +} + +// MockCortexAdminServerMockRecorder is the mock recorder for MockCortexAdminServer. +type MockCortexAdminServerMockRecorder struct { + mock *MockCortexAdminServer +} + +// NewMockCortexAdminServer creates a new mock instance. +func NewMockCortexAdminServer(ctrl *gomock.Controller) *MockCortexAdminServer { + mock := &MockCortexAdminServer{ctrl: ctrl} + mock.recorder = &MockCortexAdminServerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCortexAdminServer) EXPECT() *MockCortexAdminServerMockRecorder { + return m.recorder +} + +// AllUserStats mocks base method. +func (m *MockCortexAdminServer) AllUserStats(arg0 context.Context, arg1 *emptypb.Empty) (*cortexadmin.UserIDStatsList, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AllUserStats", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.UserIDStatsList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AllUserStats indicates an expected call of AllUserStats. +func (mr *MockCortexAdminServerMockRecorder) AllUserStats(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllUserStats", reflect.TypeOf((*MockCortexAdminServer)(nil).AllUserStats), arg0, arg1) +} + +// DeleteRule mocks base method. +func (m *MockCortexAdminServer) DeleteRule(arg0 context.Context, arg1 *cortexadmin.DeleteRuleRequest) (*emptypb.Empty, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteRule", arg0, arg1) + ret0, _ := ret[0].(*emptypb.Empty) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteRule indicates an expected call of DeleteRule. +func (mr *MockCortexAdminServerMockRecorder) DeleteRule(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRule", reflect.TypeOf((*MockCortexAdminServer)(nil).DeleteRule), arg0, arg1) +} + +// ExtractRawSeries mocks base method. +func (m *MockCortexAdminServer) ExtractRawSeries(arg0 context.Context, arg1 *cortexadmin.MatcherRequest) (*cortexadmin.QueryResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ExtractRawSeries", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.QueryResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ExtractRawSeries indicates an expected call of ExtractRawSeries. +func (mr *MockCortexAdminServerMockRecorder) ExtractRawSeries(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExtractRawSeries", reflect.TypeOf((*MockCortexAdminServer)(nil).ExtractRawSeries), arg0, arg1) +} + +// FlushBlocks mocks base method. +func (m *MockCortexAdminServer) FlushBlocks(arg0 context.Context, arg1 *emptypb.Empty) (*emptypb.Empty, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FlushBlocks", arg0, arg1) + ret0, _ := ret[0].(*emptypb.Empty) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FlushBlocks indicates an expected call of FlushBlocks. +func (mr *MockCortexAdminServerMockRecorder) FlushBlocks(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FlushBlocks", reflect.TypeOf((*MockCortexAdminServer)(nil).FlushBlocks), arg0, arg1) +} + +// GetCortexConfig mocks base method. +func (m *MockCortexAdminServer) GetCortexConfig(arg0 context.Context, arg1 *cortexadmin.ConfigRequest) (*cortexadmin.ConfigResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCortexConfig", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.ConfigResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCortexConfig indicates an expected call of GetCortexConfig. +func (mr *MockCortexAdminServerMockRecorder) GetCortexConfig(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCortexConfig", reflect.TypeOf((*MockCortexAdminServer)(nil).GetCortexConfig), arg0, arg1) +} + +// GetCortexStatus mocks base method. +func (m *MockCortexAdminServer) GetCortexStatus(arg0 context.Context, arg1 *emptypb.Empty) (*cortexadmin.CortexStatus, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCortexStatus", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.CortexStatus) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCortexStatus indicates an expected call of GetCortexStatus. +func (mr *MockCortexAdminServerMockRecorder) GetCortexStatus(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCortexStatus", reflect.TypeOf((*MockCortexAdminServer)(nil).GetCortexStatus), arg0, arg1) +} + +// GetMetricLabelSets mocks base method. +func (m *MockCortexAdminServer) GetMetricLabelSets(arg0 context.Context, arg1 *cortexadmin.LabelRequest) (*cortexadmin.MetricLabels, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMetricLabelSets", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.MetricLabels) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetMetricLabelSets indicates an expected call of GetMetricLabelSets. +func (mr *MockCortexAdminServerMockRecorder) GetMetricLabelSets(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMetricLabelSets", reflect.TypeOf((*MockCortexAdminServer)(nil).GetMetricLabelSets), arg0, arg1) +} + +// GetMetricMetadata mocks base method. +func (m *MockCortexAdminServer) GetMetricMetadata(arg0 context.Context, arg1 *cortexadmin.MetricMetadataRequest) (*cortexadmin.MetricMetadata, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMetricMetadata", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.MetricMetadata) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetMetricMetadata indicates an expected call of GetMetricMetadata. +func (mr *MockCortexAdminServerMockRecorder) GetMetricMetadata(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMetricMetadata", reflect.TypeOf((*MockCortexAdminServer)(nil).GetMetricMetadata), arg0, arg1) +} + +// GetRule mocks base method. +func (m *MockCortexAdminServer) GetRule(arg0 context.Context, arg1 *cortexadmin.GetRuleRequest) (*cortexadmin.QueryResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRule", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.QueryResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetRule indicates an expected call of GetRule. +func (mr *MockCortexAdminServerMockRecorder) GetRule(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRule", reflect.TypeOf((*MockCortexAdminServer)(nil).GetRule), arg0, arg1) +} + +// GetSeriesMetrics mocks base method. +func (m *MockCortexAdminServer) GetSeriesMetrics(arg0 context.Context, arg1 *cortexadmin.SeriesRequest) (*cortexadmin.SeriesInfoList, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSeriesMetrics", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.SeriesInfoList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSeriesMetrics indicates an expected call of GetSeriesMetrics. +func (mr *MockCortexAdminServerMockRecorder) GetSeriesMetrics(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSeriesMetrics", reflect.TypeOf((*MockCortexAdminServer)(nil).GetSeriesMetrics), arg0, arg1) +} + +// ListRules mocks base method. +func (m *MockCortexAdminServer) ListRules(arg0 context.Context, arg1 *cortexadmin.ListRulesRequest) (*cortexadmin.ListRulesResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListRules", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.ListRulesResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRules indicates an expected call of ListRules. +func (mr *MockCortexAdminServerMockRecorder) ListRules(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRules", reflect.TypeOf((*MockCortexAdminServer)(nil).ListRules), arg0, arg1) +} + +// LoadRules mocks base method. +func (m *MockCortexAdminServer) LoadRules(arg0 context.Context, arg1 *cortexadmin.LoadRuleRequest) (*emptypb.Empty, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadRules", arg0, arg1) + ret0, _ := ret[0].(*emptypb.Empty) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// LoadRules indicates an expected call of LoadRules. +func (mr *MockCortexAdminServerMockRecorder) LoadRules(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadRules", reflect.TypeOf((*MockCortexAdminServer)(nil).LoadRules), arg0, arg1) +} + +// Query mocks base method. +func (m *MockCortexAdminServer) Query(arg0 context.Context, arg1 *cortexadmin.QueryRequest) (*cortexadmin.QueryResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Query", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.QueryResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Query indicates an expected call of Query. +func (mr *MockCortexAdminServerMockRecorder) Query(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Query", reflect.TypeOf((*MockCortexAdminServer)(nil).Query), arg0, arg1) +} + +// QueryRange mocks base method. +func (m *MockCortexAdminServer) QueryRange(arg0 context.Context, arg1 *cortexadmin.QueryRangeRequest) (*cortexadmin.QueryResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "QueryRange", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.QueryResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// QueryRange indicates an expected call of QueryRange. +func (mr *MockCortexAdminServerMockRecorder) QueryRange(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueryRange", reflect.TypeOf((*MockCortexAdminServer)(nil).QueryRange), arg0, arg1) +} + +// WriteMetrics mocks base method. +func (m *MockCortexAdminServer) WriteMetrics(arg0 context.Context, arg1 *cortexadmin.WriteRequest) (*cortexadmin.WriteResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WriteMetrics", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.WriteResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// WriteMetrics indicates an expected call of WriteMetrics. +func (mr *MockCortexAdminServerMockRecorder) WriteMetrics(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteMetrics", reflect.TypeOf((*MockCortexAdminServer)(nil).WriteMetrics), arg0, arg1) +} + +// mustEmbedUnimplementedCortexAdminServer mocks base method. +func (m *MockCortexAdminServer) mustEmbedUnimplementedCortexAdminServer() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "mustEmbedUnimplementedCortexAdminServer") +} + +// mustEmbedUnimplementedCortexAdminServer indicates an expected call of mustEmbedUnimplementedCortexAdminServer. +func (mr *MockCortexAdminServerMockRecorder) mustEmbedUnimplementedCortexAdminServer() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "mustEmbedUnimplementedCortexAdminServer", reflect.TypeOf((*MockCortexAdminServer)(nil).mustEmbedUnimplementedCortexAdminServer)) +} + +// MockUnsafeCortexAdminServer is a mock of UnsafeCortexAdminServer interface. +type MockUnsafeCortexAdminServer struct { + ctrl *gomock.Controller + recorder *MockUnsafeCortexAdminServerMockRecorder +} + +// MockUnsafeCortexAdminServerMockRecorder is the mock recorder for MockUnsafeCortexAdminServer. +type MockUnsafeCortexAdminServerMockRecorder struct { + mock *MockUnsafeCortexAdminServer +} + +// NewMockUnsafeCortexAdminServer creates a new mock instance. +func NewMockUnsafeCortexAdminServer(ctrl *gomock.Controller) *MockUnsafeCortexAdminServer { + mock := &MockUnsafeCortexAdminServer{ctrl: ctrl} + mock.recorder = &MockUnsafeCortexAdminServerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockUnsafeCortexAdminServer) EXPECT() *MockUnsafeCortexAdminServerMockRecorder { + return m.recorder +} + +// mustEmbedUnimplementedCortexAdminServer mocks base method. +func (m *MockUnsafeCortexAdminServer) mustEmbedUnimplementedCortexAdminServer() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "mustEmbedUnimplementedCortexAdminServer") +} + +// mustEmbedUnimplementedCortexAdminServer indicates an expected call of mustEmbedUnimplementedCortexAdminServer. +func (mr *MockUnsafeCortexAdminServerMockRecorder) mustEmbedUnimplementedCortexAdminServer() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "mustEmbedUnimplementedCortexAdminServer", reflect.TypeOf((*MockUnsafeCortexAdminServer)(nil).mustEmbedUnimplementedCortexAdminServer)) +} diff --git a/pkg/test/mock/mockgen.yaml b/pkg/test/mock/mockgen.yaml index c5b32345e0..dbf8f37608 100644 --- a/pkg/test/mock/mockgen.yaml +++ b/pkg/test/mock/mockgen.yaml @@ -50,3 +50,16 @@ mocks: dest: pkg/test/mock/update/update.go types: - UpdateTypeHandler + - source: plugins/metrics/apis/cortexadmin/cortexadmin_grpc.pb.go + dest: pkg/test/mock/cortexadmin/cortexadmin.go + types: + - CortexAdminClient + - source: pkg/slo/backend/service.go + dest: pkg/test/mock/slo/service.go + types: + - ServiceBackend + - source: pkg/slo/backend/store.go + dest : pkg/test/mock/slo/store.go + types: + - SLOStore + diff --git a/pkg/test/mock/slo/service.go b/pkg/test/mock/slo/service.go new file mode 100644 index 0000000000..5c44d5d3f4 --- /dev/null +++ b/pkg/test/mock/slo/service.go @@ -0,0 +1,81 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: pkg/slo/backend/service.go + +// Package mock_backend is a generated GoMock package. +package mock_backend + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/rancher/opni/pkg/apis/slo/v1" +) + +// MockServiceBackend is a mock of ServiceBackend interface. +type MockServiceBackend struct { + ctrl *gomock.Controller + recorder *MockServiceBackendMockRecorder +} + +// MockServiceBackendMockRecorder is the mock recorder for MockServiceBackend. +type MockServiceBackendMockRecorder struct { + mock *MockServiceBackend +} + +// NewMockServiceBackend creates a new mock instance. +func NewMockServiceBackend(ctrl *gomock.Controller) *MockServiceBackend { + mock := &MockServiceBackend{ctrl: ctrl} + mock.recorder = &MockServiceBackendMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockServiceBackend) EXPECT() *MockServiceBackendMockRecorder { + return m.recorder +} + +// ListEvents mocks base method. +func (m *MockServiceBackend) ListEvents(ctx context.Context, req *v1.ListEventsRequest) (*v1.EventList, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListEvents", ctx, req) + ret0, _ := ret[0].(*v1.EventList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListEvents indicates an expected call of ListEvents. +func (mr *MockServiceBackendMockRecorder) ListEvents(ctx, req interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListEvents", reflect.TypeOf((*MockServiceBackend)(nil).ListEvents), ctx, req) +} + +// ListMetrics mocks base method. +func (m *MockServiceBackend) ListMetrics(ctx context.Context, req *v1.ListMetricsRequest) (*v1.MetricGroupList, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListMetrics", ctx, req) + ret0, _ := ret[0].(*v1.MetricGroupList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListMetrics indicates an expected call of ListMetrics. +func (mr *MockServiceBackendMockRecorder) ListMetrics(ctx, req interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListMetrics", reflect.TypeOf((*MockServiceBackend)(nil).ListMetrics), ctx, req) +} + +// ListServices mocks base method. +func (m *MockServiceBackend) ListServices(ctx context.Context, req *v1.ListServicesRequest) (*v1.ServiceList, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListServices", ctx, req) + ret0, _ := ret[0].(*v1.ServiceList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListServices indicates an expected call of ListServices. +func (mr *MockServiceBackendMockRecorder) ListServices(ctx, req interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListServices", reflect.TypeOf((*MockServiceBackend)(nil).ListServices), ctx, req) +} diff --git a/pkg/test/mock/slo/store.go b/pkg/test/mock/slo/store.go new file mode 100644 index 0000000000..757d924db7 --- /dev/null +++ b/pkg/test/mock/slo/store.go @@ -0,0 +1,143 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: pkg/slo/backend/store.go + +// Package mock_backend is a generated GoMock package. +package mock_backend + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/rancher/opni/pkg/apis/core/v1" + v10 "github.com/rancher/opni/pkg/apis/slo/v1" +) + +// MockSLOStore is a mock of SLOStore interface. +type MockSLOStore struct { + ctrl *gomock.Controller + recorder *MockSLOStoreMockRecorder +} + +// MockSLOStoreMockRecorder is the mock recorder for MockSLOStore. +type MockSLOStoreMockRecorder struct { + mock *MockSLOStore +} + +// NewMockSLOStore creates a new mock instance. +func NewMockSLOStore(ctrl *gomock.Controller) *MockSLOStore { + mock := &MockSLOStore{ctrl: ctrl} + mock.recorder = &MockSLOStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSLOStore) EXPECT() *MockSLOStoreMockRecorder { + return m.recorder +} + +// Clone mocks base method. +func (m *MockSLOStore) Clone(ctx context.Context, clone *v10.SLOData) (*v1.Reference, *v10.SLOData, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Clone", ctx, clone) + ret0, _ := ret[0].(*v1.Reference) + ret1, _ := ret[1].(*v10.SLOData) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// Clone indicates an expected call of Clone. +func (mr *MockSLOStoreMockRecorder) Clone(ctx, clone interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Clone", reflect.TypeOf((*MockSLOStore)(nil).Clone), ctx, clone) +} + +// Create mocks base method. +func (m *MockSLOStore) Create(ctx context.Context, req *v10.CreateSLORequest) (*v1.Reference, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", ctx, req) + ret0, _ := ret[0].(*v1.Reference) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Create indicates an expected call of Create. +func (mr *MockSLOStoreMockRecorder) Create(ctx, req interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockSLOStore)(nil).Create), ctx, req) +} + +// Delete mocks base method. +func (m *MockSLOStore) Delete(ctx context.Context, existing *v10.SLOData) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", ctx, existing) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockSLOStoreMockRecorder) Delete(ctx, existing interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockSLOStore)(nil).Delete), ctx, existing) +} + +// MultiClusterClone mocks base method. +func (m *MockSLOStore) MultiClusterClone(ctx context.Context, slo *v10.SLOData, toClusters []*v1.Reference) ([]*v1.Reference, []*v10.SLOData, []error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MultiClusterClone", ctx, slo, toClusters) + ret0, _ := ret[0].([]*v1.Reference) + ret1, _ := ret[1].([]*v10.SLOData) + ret2, _ := ret[2].([]error) + return ret0, ret1, ret2 +} + +// MultiClusterClone indicates an expected call of MultiClusterClone. +func (mr *MockSLOStoreMockRecorder) MultiClusterClone(ctx, slo, toClusters interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MultiClusterClone", reflect.TypeOf((*MockSLOStore)(nil).MultiClusterClone), ctx, slo, toClusters) +} + +// Preview mocks base method. +func (m *MockSLOStore) Preview(ctx context.Context, s *v10.CreateSLORequest) (*v10.SLOPreviewResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Preview", ctx, s) + ret0, _ := ret[0].(*v10.SLOPreviewResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Preview indicates an expected call of Preview. +func (mr *MockSLOStoreMockRecorder) Preview(ctx, s interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Preview", reflect.TypeOf((*MockSLOStore)(nil).Preview), ctx, s) +} + +// Status mocks base method. +func (m *MockSLOStore) Status(ctx context.Context, existing *v10.SLOData) (*v10.SLOStatus, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Status", ctx, existing) + ret0, _ := ret[0].(*v10.SLOStatus) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Status indicates an expected call of Status. +func (mr *MockSLOStoreMockRecorder) Status(ctx, existing interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Status", reflect.TypeOf((*MockSLOStore)(nil).Status), ctx, existing) +} + +// Update mocks base method. +func (m *MockSLOStore) Update(ctx context.Context, incoming, existing *v10.SLOData) (*v10.SLOData, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Update", ctx, incoming, existing) + ret0, _ := ret[0].(*v10.SLOData) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Update indicates an expected call of Update. +func (mr *MockSLOStoreMockRecorder) Update(ctx, incoming, existing interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockSLOStore)(nil).Update), ctx, incoming, existing) +} diff --git a/plugins/alerting/pkg/alerting/plugin.go b/plugins/alerting/pkg/alerting/plugin.go index ca17f872df..07a7a7cf7d 100644 --- a/plugins/alerting/pkg/alerting/plugin.go +++ b/plugins/alerting/pkg/alerting/plugin.go @@ -10,6 +10,7 @@ import ( "github.com/rancher/opni/pkg/storage" "github.com/rancher/opni/plugins/alerting/apis/alertops" metricsExporter "github.com/rancher/opni/plugins/alerting/pkg/alerting/metrics" + "github.com/rancher/opni/plugins/alerting/pkg/alerting/slo/v1" "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" "github.com/rancher/opni/plugins/metrics/apis/cortexops" metricsdk "go.opentelemetry.io/otel/sdk/metric" @@ -31,6 +32,7 @@ import ( capabilityv1 "github.com/rancher/opni/pkg/apis/capability/v1" managementv1 "github.com/rancher/opni/pkg/apis/management/v1" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" "github.com/rancher/opni/pkg/logger" managementext "github.com/rancher/opni/pkg/plugins/apis/apiextensions/management" streamext "github.com/rancher/opni/pkg/plugins/apis/apiextensions/stream" @@ -47,6 +49,7 @@ func (p *Plugin) Components() []server.ServerComponent { p.NotificationServerComponent, p.EndpointServerComponent, p.AlarmServerComponent, + p.SLOServerComponent, } } @@ -73,6 +76,7 @@ type Plugin struct { cortexOpsClient future.Future[cortexops.CortexOpsClient] natsConn future.Future[*nats.Conn] js future.Future[nats.JetStreamContext] + sloStorage future.Future[slo.StorageAPIs] globalWatchers management.ConditionWatcher gatewayConfig future.Future[*v1beta1.GatewayConfig] @@ -82,6 +86,7 @@ type Plugin struct { *notifications.NotificationServerComponent *endpoints.EndpointServerComponent *alarms.AlarmServerComponent + *slo.SLOServerComponent node node_backend.AlertingNodeBackend @@ -120,6 +125,7 @@ func NewPlugin(ctx context.Context) *Plugin { cortexOpsClient: future.New[cortexops.CortexOpsClient](), natsConn: future.New[*nats.Conn](), js: future.New[nats.JetStreamContext](), + sloStorage: future.New[slo.StorageAPIs](), gatewayConfig: future.New[*v1beta1.GatewayConfig](), @@ -156,6 +162,10 @@ func NewPlugin(ctx context.Context) *Plugin { p.NotificationServerComponent, ) + p.SLOServerComponent = slo.NewSLOServerComponent( + p.logger.With("component", "slo"), + ) + future.Wait4( p.storageBackend, p.mgmtClient, @@ -196,6 +206,16 @@ func NewPlugin(ctx context.Context) *Plugin { ) }) + future.Wait2(p.mgmtClient, p.sloStorage, func( + mgmtClient managementv1.ManagementClient, + sloStorage slo.StorageAPIs, + ) { + p.SLOServerComponent.Initialize(slo.SLOServerConfiguration{ + MgmtClient: mgmtClient, + Storage: sloStorage, + }) + }) + future.Wait5(p.js, p.storageClientSet, p.mgmtClient, p.adminClient, p.cortexOpsClient, func( js nats.JetStreamContext, @@ -267,6 +287,10 @@ func Scheme(ctx context.Context) meta.Scheme { &node.NodeAlertingCapability_ServiceDesc, &p.node, ), + util.PackService( + &slov1.SLO_ServiceDesc, + p.SLOServerComponent, + ), ), ) diff --git a/plugins/alerting/pkg/alerting/slo/v1/component.go b/plugins/alerting/pkg/alerting/slo/v1/component.go new file mode 100644 index 0000000000..9d29d23626 --- /dev/null +++ b/plugins/alerting/pkg/alerting/slo/v1/component.go @@ -0,0 +1,96 @@ +package slo + +import ( + "context" + "sync" + + "github.com/rancher/opni/pkg/slo/backend" + "github.com/rancher/opni/pkg/storage" + + "github.com/rancher/opni/pkg/alerting/server" + alertingSync "github.com/rancher/opni/pkg/alerting/server/sync" + managementv1 "github.com/rancher/opni/pkg/apis/management/v1" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/rancher/opni/pkg/util" + "github.com/rancher/opni/pkg/util/future" + "go.uber.org/zap" +) + +type SLOServerComponent struct { + slov1.UnsafeSLOServer + util.Initializer + logger *zap.SugaredLogger + storage future.Future[StorageAPIs] + mgmtClient future.Future[managementv1.ManagementClient] +} + +var _ server.ServerComponent = (*SLOServerComponent)(nil) +var _ slov1.SLOServer = (*SLOServerComponent)(nil) + +type StorageAPIs struct { + SLOs storage.KeyValueStoreT[*slov1.SLOData] + Services storage.KeyValueStoreT[*slov1.Service] + Metrics storage.KeyValueStoreT[*slov1.Metric] +} + +func NewSLOServerComponent( + logger *zap.SugaredLogger, +) *SLOServerComponent { + return &SLOServerComponent{ + logger: logger, + storage: future.New[StorageAPIs](), + mgmtClient: future.New[managementv1.ManagementClient](), + } +} + +type SLOServerConfiguration struct { + Storage StorageAPIs + MgmtClient managementv1.ManagementClient +} + +func (s *SLOServerComponent) Name() string { + return "notification" +} + +func (s *SLOServerComponent) Status() server.Status { + return server.Status{ + Running: s.Initialized(), + } +} + +func (s *SLOServerComponent) Ready() bool { + return s.Initialized() +} + +func (s *SLOServerComponent) Healthy() bool { + return s.Initialized() +} + +func (s *SLOServerComponent) SetConfig(_ server.Config) { +} + +func (s *SLOServerComponent) Sync(_ context.Context, _ alertingSync.SyncInfo) error { + return nil +} + +func (s *SLOServerComponent) Initialize(conf SLOServerConfiguration) { + s.InitOnce(func() { + s.mgmtClient.Set(conf.MgmtClient) + s.storage.Set(conf.Storage) + }) +} + +var datasourceMu sync.RWMutex + +var ( + datasources = make(map[string]backend.SLODatasource) +) + +func RegisterDatasource( + datasourceName string, + datasourceImpl backend.SLODatasource, +) { + defer datasourceMu.Unlock() + datasourceMu.Lock() + datasources[datasourceName] = datasourceImpl +} diff --git a/plugins/alerting/pkg/alerting/slo/v1/server.go b/plugins/alerting/pkg/alerting/slo/v1/server.go new file mode 100644 index 0000000000..50f1353b13 --- /dev/null +++ b/plugins/alerting/pkg/alerting/slo/v1/server.go @@ -0,0 +1,311 @@ +package slo + +import ( + "context" + "fmt" + "path" + "time" + + corev1 "github.com/rancher/opni/pkg/apis/core/v1" + managementv1 "github.com/rancher/opni/pkg/apis/management/v1" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/rancher/opni/pkg/slo/shared" + "github.com/rancher/opni/pkg/storage" + "github.com/rancher/opni/pkg/validation" + "github.com/samber/lo" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func list[T proto.Message](ctx context.Context, kvc storage.KeyValueStoreT[T], prefix string) ([]T, error) { + keys, err := kvc.ListKeys(ctx, prefix) + if err != nil { + return nil, err + } + items := make([]T, len(keys)) + for i, key := range keys { + item, err := kvc.Get(ctx, key) + if err != nil { + return nil, err + } + items[i] = item + } + return items, nil +} + +func checkDatasource(datasource string) error { + datasourceMu.RLock() + defer datasourceMu.RUnlock() + if _, ok := datasources[datasource]; !ok { + return shared.ErrInvalidDatasource + } + return nil +} + +func (s *SLOServerComponent) GetSLO(ctx context.Context, ref *corev1.Reference) (*slov1.SLOData, error) { + return s.storage.Get().SLOs.Get(ctx, path.Join("/slos", ref.Id)) +} + +func (s *SLOServerComponent) ListSLOs(ctx context.Context, _ *emptypb.Empty) (*slov1.ServiceLevelObjectiveList, error) { + items, err := list(ctx, s.storage.Get().SLOs, "/slos") + if err != nil { + return nil, err + } + return &slov1.ServiceLevelObjectiveList{ + Items: items, + }, nil +} + +func (s *SLOServerComponent) CreateSLO(ctx context.Context, req *slov1.CreateSLORequest) (*corev1.Reference, error) { + if err := req.Validate(); err != nil { + return nil, err + } + clusterList, err := s.mgmtClient.Get().ListClusters(ctx, &managementv1.ListClustersRequest{}) + if err != nil { + return nil, err + } + if !clusterList.Has(req.Slo.ClusterId) { + return nil, validation.Error("invalid cluster : not found") + } + datasourceMu.RLock() + sloStore := datasources[req.GetSlo().GetDatasource()] + datasourceMu.RUnlock() + + if err := sloStore.Precondition(ctx, &corev1.Reference{Id: req.Slo.ClusterId}); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + + id, err := sloStore.Create(ctx, req) + if err != nil { + return nil, err + } + sloData := &slov1.SLOData{ + Id: id.Id, + SLO: req.GetSlo(), + CreatedAt: timestamppb.New(time.Now()), + } + if err := s.storage.Get().SLOs.Put(ctx, path.Join("/slos", id.Id), sloData); err != nil { + return nil, err + } + return id, nil +} + +func (s *SLOServerComponent) UpdateSLO(ctx context.Context, req *slov1.SLOData) (*emptypb.Empty, error) { + if err := req.Validate(); err != nil { + return nil, err + } + clusterList, err := s.mgmtClient.Get().ListClusters(ctx, &managementv1.ListClustersRequest{}) + if err != nil { + return nil, err + } + validCluster := false + for _, clusterListItem := range clusterList.Items { + if clusterListItem.Id == req.SLO.ClusterId { + validCluster = true + break + } + } + if !validCluster { + return nil, validation.Error("invalid cluster") + } + existing, err := s.storage.Get().SLOs.Get(ctx, path.Join("/slos", req.Id)) + if err != nil { + return nil, err + } + sloStore := datasources[req.GetSLO().GetDatasource()] + if err := sloStore.Precondition(ctx, &corev1.Reference{Id: req.SLO.ClusterId}); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + updatedSLO, err := sloStore.Update(ctx, req, existing) + if err != nil { // exit when update fails + return nil, err + } + // Merge when everything else is done + if err := s.storage.Get().SLOs.Put(ctx, path.Join("/slos", req.Id), updatedSLO); err != nil { + return nil, err + } + return &emptypb.Empty{}, nil +} + +func (s *SLOServerComponent) DeleteSLO(ctx context.Context, req *corev1.Reference) (*emptypb.Empty, error) { + lg := s.logger + existing, err := s.storage.Get().SLOs.Get(ctx, path.Join("/slos", req.Id)) + if err != nil { + lg.With("delete slo", req.Id).Error("failed to get slo to delete in K,V store") + return nil, err + } + if err := checkDatasource(existing.SLO.GetDatasource()); err != nil { + return nil, err + } + sloStore := datasources[existing.SLO.GetDatasource()] + if err := sloStore.Precondition(ctx, &corev1.Reference{Id: existing.SLO.ClusterId}); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + err = sloStore.Delete(ctx, existing) + if err != nil { + return nil, err + } + if err := s.storage.Get().SLOs.Delete(ctx, path.Join("/slos", req.Id)); err != nil { + return nil, err + } + return &emptypb.Empty{}, nil +} + +func (s *SLOServerComponent) CloneSLO(ctx context.Context, ref *corev1.Reference) (*slov1.SLOData, error) { + existing, err := s.storage.Get().SLOs.Get(ctx, path.Join("/slos", ref.Id)) + if err != nil { + return nil, err + } + if err := existing.Validate(); err != nil { + return nil, err + } + if err := checkDatasource(existing.SLO.GetDatasource()); err != nil { + return nil, err + } + sloStore := datasources[existing.SLO.GetDatasource()] + if err := sloStore.Precondition(ctx, &corev1.Reference{Id: existing.SLO.ClusterId}); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + newId, newData, anyError := sloStore.Clone(ctx, existing) + newData.CreatedAt = timestamppb.New(time.Now()) + if anyError != nil { + return nil, anyError + } + if err := s.storage.Get().SLOs.Put(ctx, path.Join("/slos", newId.Id), newData); err != nil { + return nil, err + } + return newData, nil +} + +func (s *SLOServerComponent) CloneToClusters(ctx context.Context, req *slov1.MultiClusterSLO) (*slov1.MultiClusterFailures, error) { + existing, err := s.storage.Get().SLOs.Get(ctx, path.Join("/slos", req.CloneId.Id)) + if err != nil { + return nil, err + } + if err := existing.Validate(); err != nil { + return nil, err + } + if err := checkDatasource(existing.SLO.GetDatasource()); err != nil { + return nil, err + } + sloStore := datasources[existing.SLO.GetDatasource()] + for _, cluster := range req.Clusters { + if err := sloStore.Precondition(ctx, cluster); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + } + failures := []string{} + createdIds, createdData, errors := sloStore.MultiClusterClone(ctx, existing, req.Clusters) + createdAt := timestamppb.New(time.Now()) + for idx, err := range errors { + if err != nil { + failures = append(failures, err.Error()) + } else { + createdData[idx].CreatedAt = createdAt + if err := s.storage.Get().SLOs.Put( + ctx, + path.Join("/slos", createdIds[idx].Id), + createdData[idx], + ); err != nil { + failures = append(failures, fmt.Sprintf("failed to persist SLO configuration : %s", err.Error())) + } + } + } + return &slov1.MultiClusterFailures{ + Failures: failures, + }, nil +} + +func (s *SLOServerComponent) GetSLOStatus(ctx context.Context, ref *corev1.Reference) (*slov1.SLOStatus, error) { + existing, err := s.storage.Get().SLOs.Get(ctx, path.Join("/slos", ref.Id)) + if err != nil { + return nil, err + } + if err := checkDatasource(existing.SLO.GetDatasource()); err != nil { + return nil, err + } + sloStore := datasources[existing.SLO.GetDatasource()] + if err := sloStore.Precondition(ctx, &corev1.Reference{Id: existing.SLO.ClusterId}); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + status, err := sloStore.Status(ctx, existing) + return status, err +} + +func (s *SLOServerComponent) Preview(ctx context.Context, req *slov1.CreateSLORequest) (*slov1.SLOPreviewResponse, error) { + if err := req.Validate(); err != nil { + return nil, err + } + if err := checkDatasource(req.GetSlo().GetDatasource()); err != nil { + return nil, err + } + sloStore := datasources[req.GetSlo().GetDatasource()] + if err := sloStore.Precondition(ctx, &corev1.Reference{Id: req.Slo.ClusterId}); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + preview, err := sloStore.Preview(ctx, req) + return preview, err +} + +// -------- Service Discovery --------- + +func (s *SLOServerComponent) ListServices(ctx context.Context, req *slov1.ListServicesRequest) (*slov1.ServiceList, error) { + err := checkDatasource(req.Datasource) + if err != nil { + return nil, shared.ErrInvalidDatasource + } + datasourceMu.RLock() + backend := datasources[req.Datasource] + datasourceMu.RUnlock() + + if err := backend.Precondition(ctx, &corev1.Reference{Id: req.ClusterId}); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + return backend.ListServices(ctx, req) +} + +func (s *SLOServerComponent) ListMetrics(ctx context.Context, req *slov1.ListMetricsRequest) (*slov1.MetricGroupList, error) { + err := checkDatasource(req.Datasource) + if err != nil { + return nil, shared.ErrInvalidDatasource + } + datasourceMu.RLock() + backend := datasources[req.Datasource] + datasourceMu.RUnlock() + + if err := backend.Precondition(ctx, &corev1.Reference{Id: req.ClusterId}); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + return backend.ListMetrics(ctx, req) +} + +func (s *SLOServerComponent) ListEvents(ctx context.Context, req *slov1.ListEventsRequest) (*slov1.EventList, error) { + // fetch labels & their label values for the given cluster & service + if err := req.Validate(); err != nil { + return nil, err + } + datasource := req.GetDatasource() + if err := checkDatasource(datasource); err != nil { + return nil, shared.ErrInvalidDatasource + } + datasourceMu.RLock() + backend := datasources[req.Datasource] + datasourceMu.RUnlock() + + if err := backend.Precondition(ctx, &corev1.Reference{Id: req.ClusterId}); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + return backend.ListEvents(ctx, req) +} + +func (s *SLOServerComponent) ListDatasources(_ context.Context, _ *emptypb.Empty) (*slov1.DatasourceInfo, error) { + datasourceMu.RLock() + datasources := lo.Keys(datasources) + defer datasourceMu.RUnlock() + return &slov1.DatasourceInfo{ + Items: datasources, + }, nil +} diff --git a/plugins/alerting/pkg/alerting/system.go b/plugins/alerting/pkg/alerting/system.go index e37dd3158d..23888a0a94 100644 --- a/plugins/alerting/pkg/alerting/system.go +++ b/plugins/alerting/pkg/alerting/system.go @@ -10,10 +10,15 @@ import ( "time" "github.com/rancher/opni/pkg/caching" + "github.com/rancher/opni/pkg/capabilities" + "github.com/rancher/opni/pkg/capabilities/wellknown" "github.com/rancher/opni/pkg/management" + "github.com/rancher/opni/pkg/slo/backend" + "github.com/rancher/opni/pkg/slo/backend/metrics" "github.com/rancher/opni/pkg/storage" "github.com/rancher/opni/plugins/alerting/apis/alertops" "github.com/rancher/opni/plugins/alerting/pkg/alerting/alarms/v1" + "github.com/rancher/opni/plugins/alerting/pkg/alerting/slo/v1" "github.com/rancher/opni/plugins/alerting/pkg/node_backend" "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" "github.com/rancher/opni/plugins/metrics/apis/cortexops" @@ -26,6 +31,7 @@ import ( "github.com/rancher/opni/pkg/alerting/server" capabilityv1 "github.com/rancher/opni/pkg/apis/capability/v1" + corev1 "github.com/rancher/opni/pkg/apis/core/v1" managementv1 "github.com/rancher/opni/pkg/apis/management/v1" "github.com/rancher/opni/pkg/config/v1beta1" "github.com/rancher/opni/pkg/machinery" @@ -37,6 +43,7 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/emptypb" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" _ "github.com/rancher/opni/pkg/storage/etcd" _ "github.com/rancher/opni/pkg/storage/jetstream" ) @@ -96,6 +103,12 @@ func (p *Plugin) UseKeyValueStore(client system.KeyValueStoreClient) { NodeCapabilitySpecs: storage.NewKeyValueStoreWithPrefix(system.NewKVStoreClient[*node.AlertingCapabilitySpec](client), "/alerting/config/capability/nodes"), }) + p.sloStorage.Set(slo.StorageAPIs{ + SLOs: system.NewKVStoreClient[*slov1.SLOData](client), + Services: system.NewKVStoreClient[*slov1.Service](client), + Metrics: system.NewKVStoreClient[*slov1.Metric](client), + }) + var ( nc *nats.Conn err error @@ -193,6 +206,8 @@ func (p *Plugin) UseAPIExtensions(intf system.ExtensionClientInterface) { } p.adminClient.Set(cortexadmin.NewCortexAdminClient(cc)) p.cortexOpsClient.Set(cortexops.NewCortexOpsClient(cc)) + + p.registerMetricsDatasource() } func (p *Plugin) UseNodeManagerClient(client capabilityv1.NodeManagerClient) { @@ -200,6 +215,42 @@ func (p *Plugin) UseNodeManagerClient(client capabilityv1.NodeManagerClient) { <-p.ctx.Done() } +func (p *Plugin) registerMetricsDatasource() { + metricsProvider := metrics.NewProvider(p.logger.With("component", "slo")) + metricsProvider.Initialize(p.adminClient.Get()) + metricsStoreProvider := metrics.NewMetricsSLOStoreProvider(p.logger.With("component", "slo")) + metricsStoreProvider.Initialize(p.adminClient.Get()) + + metricsDatasource := backend.NewSLODatasource( + metrics.NewMetricsSLOStore( + metricsStoreProvider, + ), + metrics.NewBackend( + metricsProvider, + ), + func(ctx context.Context, clusterId *corev1.Reference) error { + mgmtClient, err := p.mgmtClient.GetContext(ctx) + if err != nil { + return err + } + cluster, err := mgmtClient.GetCluster(ctx, &corev1.Reference{ + Id: clusterId.Id, + }) + if err != nil { + return err + } + if !capabilities.Has(cluster, capabilities.Cluster(wellknown.CapabilityMetrics)) { + return fmt.Errorf("requires metrics capability to be installed on cluster %s", cluster.GetId()) + } + return nil + }, + ) + slo.RegisterDatasource( + "monitoring", + metricsDatasource, + ) +} + func (p *Plugin) handleDriverNotifications() { for { select { diff --git a/plugins/slo/apis/slo/compat.go b/plugins/slo/apis/slo/compat.go deleted file mode 100644 index f0da933ad3..0000000000 --- a/plugins/slo/apis/slo/compat.go +++ /dev/null @@ -1,11 +0,0 @@ -package slo - -import ( - "github.com/kralicky/ragu/compat" - _ "k8s.io/api/core/v1" -) - -func init() { - compat.LoadGogoFileDescriptor("k8s.io/api/core/v1/generated.proto") - compat.LoadGogoFileDescriptor("k8s.io/apimachinery/pkg/api/resource/generated.proto") -} diff --git a/plugins/slo/main.go b/plugins/slo/main.go deleted file mode 100644 index 4c572fbcb3..0000000000 --- a/plugins/slo/main.go +++ /dev/null @@ -1,16 +0,0 @@ -package main - -import ( - "github.com/rancher/opni/pkg/plugins" - "github.com/rancher/opni/pkg/plugins/meta" - "github.com/rancher/opni/plugins/slo/pkg/slo" -) - -func main() { - m := plugins.Main{ - Modes: meta.ModeSet{ - meta.ModeGateway: slo.Scheme, - }, - } - m.Exec() -} diff --git a/plugins/slo/pkg/slo/api.go b/plugins/slo/pkg/slo/api.go deleted file mode 100644 index 915c3bc675..0000000000 --- a/plugins/slo/pkg/slo/api.go +++ /dev/null @@ -1,266 +0,0 @@ -// API implementation - -package slo - -import ( - "context" - "path" - "time" - - corev1 "github.com/rancher/opni/pkg/apis/core/v1" - managementv1 "github.com/rancher/opni/pkg/apis/management/v1" - "github.com/rancher/opni/pkg/slo/shared" - "github.com/rancher/opni/pkg/storage" - "github.com/rancher/opni/pkg/validation" - sloapi "github.com/rancher/opni/plugins/slo/apis/slo" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/emptypb" - "google.golang.org/protobuf/types/known/timestamppb" -) - -func list[T proto.Message](ctx context.Context, kvc storage.KeyValueStoreT[T], prefix string) ([]T, error) { - keys, err := kvc.ListKeys(ctx, prefix) - if err != nil { - return nil, err - } - items := make([]T, len(keys)) - for i, key := range keys { - item, err := kvc.Get(ctx, key) - if err != nil { - return nil, err - } - items[i] = item - } - return items, nil -} - -func checkDatasource(datasource string) error { - if _, ok := datasourceToSLO[datasource]; !ok { - return shared.ErrInvalidDatasource - } - if _, ok := datasourceToService[datasource]; !ok { - return shared.ErrInvalidDatasource - } - return nil -} - -func (p *Plugin) GetSLO(ctx context.Context, ref *corev1.Reference) (*sloapi.SLOData, error) { - return p.storage.Get().SLOs.Get(ctx, path.Join("/slos", ref.Id)) -} - -func (p *Plugin) ListSLOs(ctx context.Context, _ *emptypb.Empty) (*sloapi.ServiceLevelObjectiveList, error) { - items, err := list(ctx, p.storage.Get().SLOs, "/slos") - if err != nil { - return nil, err - } - return &sloapi.ServiceLevelObjectiveList{ - Items: items, - }, nil -} - -func (p *Plugin) CreateSLO(ctx context.Context, slorequest *sloapi.CreateSLORequest) (*corev1.Reference, error) { - if err := slorequest.Validate(); err != nil { - return nil, err - } - clusterList, err := p.mgmtClient.Get().ListClusters(ctx, &managementv1.ListClustersRequest{}) - if err != nil { - return nil, err - } - validCluster := false - for _, clusterListItem := range clusterList.Items { - if clusterListItem.Id == slorequest.Slo.ClusterId { - validCluster = true - break - } - } - if !validCluster { - return nil, validation.Error("invalid cluster") - } - sloStore := datasourceToSLO[slorequest.GetSlo().GetDatasource()].WithCurrentRequest(ctx, slorequest) - id, err := sloStore.Create() - if err != nil { - return nil, err - } - sloData := &sloapi.SLOData{ - Id: id.Id, - SLO: slorequest.GetSlo(), - CreatedAt: timestamppb.New(time.Now()), - } - if err := p.storage.Get().SLOs.Put(ctx, path.Join("/slos", id.Id), sloData); err != nil { - return nil, err - } - return id, nil -} - -func (p *Plugin) UpdateSLO(ctx context.Context, req *sloapi.SLOData) (*emptypb.Empty, error) { - if err := req.Validate(); err != nil { - return nil, err - } - clusterList, err := p.mgmtClient.Get().ListClusters(ctx, &managementv1.ListClustersRequest{}) - if err != nil { - return nil, err - } - validCluster := false - for _, clusterListItem := range clusterList.Items { - if clusterListItem.Id == req.SLO.ClusterId { - validCluster = true - break - } - } - if !validCluster { - return nil, validation.Error("invalid cluster") - } - existing, err := p.storage.Get().SLOs.Get(ctx, path.Join("/slos", req.Id)) - if err != nil { - return nil, err - } - sloStore := datasourceToSLO[req.GetSLO().GetDatasource()].WithCurrentRequest(ctx, req) - updatedSLO, err := sloStore.Update(existing) - if err != nil { // exit when update fails - return nil, err - } - // Merge when everything else is done - if err := p.storage.Get().SLOs.Put(ctx, path.Join("/slos", req.Id), updatedSLO); err != nil { - return nil, err - } - return &emptypb.Empty{}, nil -} - -func (p *Plugin) DeleteSLO(ctx context.Context, req *corev1.Reference) (*emptypb.Empty, error) { - lg := p.logger - existing, err := p.storage.Get().SLOs.Get(ctx, path.Join("/slos", req.Id)) - if err != nil { - lg.With("delete slo", req.Id).Error("failed to get slo to delete in K,V store") - return nil, err - } - if err := checkDatasource(existing.SLO.GetDatasource()); err != nil { - return nil, err - } - sloStore := datasourceToSLO[existing.SLO.GetDatasource()].WithCurrentRequest(ctx, req) - err = sloStore.Delete(existing) - if err != nil { - return nil, err - } - if err := p.storage.Get().SLOs.Delete(ctx, path.Join("/slos", req.Id)); err != nil { - return nil, err - } - return &emptypb.Empty{}, nil -} - -func (p *Plugin) CloneSLO(ctx context.Context, ref *corev1.Reference) (*sloapi.SLOData, error) { - existing, err := p.storage.Get().SLOs.Get(ctx, path.Join("/slos", ref.Id)) - if err != nil { - return nil, err - } - if err := existing.Validate(); err != nil { - return nil, err - } - if err := checkDatasource(existing.SLO.GetDatasource()); err != nil { - return nil, err - } - sloStore := datasourceToSLO[existing.SLO.GetDatasource()].WithCurrentRequest(ctx, ref) - newId, newData, anyError := sloStore.Clone(existing) - newData.CreatedAt = timestamppb.New(time.Now()) - if anyError != nil { - return nil, anyError - } - if err := p.storage.Get().SLOs.Put(ctx, path.Join("/slos", newId.Id), newData); err != nil { - return nil, err - } - return newData, nil -} - -func (p *Plugin) CloneToClusters(ctx context.Context, req *sloapi.MultiClusterSLO) (*sloapi.MultiClusterFailures, error) { - existing, err := p.storage.Get().SLOs.Get(ctx, path.Join("/slos", req.CloneId.Id)) - if err != nil { - return nil, err - } - if err := existing.Validate(); err != nil { - return nil, err - } - if err := checkDatasource(existing.SLO.GetDatasource()); err != nil { - return nil, err - } - sloStore := datasourceToSLO[existing.SLO.GetDatasource()].WithCurrentRequest(ctx, req) - svcBackend := datasourceToService[existing.SLO.GetDatasource()] // with current request is set in multi cluster clone - failures := []string{} - createdIds, createdData, errors := sloStore.MultiClusterClone(existing, req.Clusters, svcBackend) - createdAt := timestamppb.New(time.Now()) - for idx, err := range errors { - if err != nil { - failures = append(failures, err.Error()) - } else { - createdData[idx].CreatedAt = createdAt - if err := p.storage.Get().SLOs.Put( - ctx, - path.Join("/slos", createdIds[idx].Id), - createdData[idx], - ); err != nil { - //FIXME: maybe don't do this - return nil, err - } - } - } - return &sloapi.MultiClusterFailures{ - Failures: failures, - }, nil -} - -func (p *Plugin) Status(ctx context.Context, ref *corev1.Reference) (*sloapi.SLOStatus, error) { - existing, err := p.storage.Get().SLOs.Get(ctx, path.Join("/slos", ref.Id)) - if err != nil { - return nil, err - } - if err := checkDatasource(existing.SLO.GetDatasource()); err != nil { - return nil, err - } - sloStore := datasourceToSLO[existing.SLO.GetDatasource()].WithCurrentRequest(ctx, ref) - status, err := sloStore.Status(existing) - return status, err -} - -func (p *Plugin) Preview(ctx context.Context, req *sloapi.CreateSLORequest) (*sloapi.SLOPreviewResponse, error) { - if err := req.Validate(); err != nil { - return nil, err - } - slo := CreateSLORequestToStruct(req) - if err := checkDatasource(req.GetSlo().GetDatasource()); err != nil { - return nil, err - } - sloStore := datasourceToSLO[req.GetSlo().GetDatasource()].WithCurrentRequest(ctx, req) - return sloStore.Preview(slo) -} - -// -------- Service Discovery --------- - -func (p *Plugin) ListServices(ctx context.Context, req *sloapi.ListServicesRequest) (*sloapi.ServiceList, error) { - //lg := p.logger - err := checkDatasource(req.Datasource) - if err != nil { - return nil, shared.ErrInvalidDatasource - } - backend := datasourceToService[req.Datasource].WithCurrentRequest(ctx, req) - return backend.ListServices() -} - -func (p *Plugin) ListMetrics(ctx context.Context, req *sloapi.ListMetricsRequest) (*sloapi.MetricGroupList, error) { - err := checkDatasource(req.Datasource) - if err != nil { - return nil, shared.ErrInvalidDatasource - } - backend := datasourceToService[req.Datasource].WithCurrentRequest(ctx, req) - return backend.ListMetrics() -} - -func (p *Plugin) ListEvents(ctx context.Context, req *sloapi.ListEventsRequest) (*sloapi.EventList, error) { - // fetch labels & their label values for the given cluster & service - if err := req.Validate(); err != nil { - return nil, err - } - datasource := req.GetDatasource() - if err := checkDatasource(datasource); err != nil { - return nil, shared.ErrInvalidDatasource - } - backend := datasourceToService[datasource].WithCurrentRequest(ctx, req) - return backend.ListEvents() -} diff --git a/plugins/slo/pkg/slo/cortex_info.go b/plugins/slo/pkg/slo/cortex_info.go deleted file mode 100644 index 6878d34ab7..0000000000 --- a/plugins/slo/pkg/slo/cortex_info.go +++ /dev/null @@ -1,224 +0,0 @@ -package slo - -import ( - "context" - "fmt" - "time" - - "emperror.dev/errors" - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/rulefmt" - "github.com/rancher/opni/pkg/metrics/compat" - "go.uber.org/zap" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "google.golang.org/protobuf/types/known/durationpb" - "google.golang.org/protobuf/types/known/timestamppb" - "gopkg.in/yaml.v3" -) - -var instantMaskDisabled = true - -func createGrafanaSLOMask(ctx context.Context, p *Plugin, clusterId string, ruleId string) error { - p.logger.With("sloId", ruleId, "clusterId", clusterId).Debugf("creating grafana mask") - if !instantMaskDisabled { - _, err := p.adminClient.Get().WriteMetrics(ctx, &cortexadmin.WriteRequest{ - ClusterID: clusterId, - Timeseries: []*cortexadmin.TimeSeries{ - { - Labels: []*cortexadmin.Label{ - { - Name: "__name__", //unique identifier for metrics - Value: grafana_delete_mask, - }, - { - Name: slo_uuid, - Value: ruleId, - }, - }, - }, - }, - }) - return err - } - return nil -} - -func tryApplyThenDeleteCortexRules( - ctx context.Context, - p *Plugin, - lg *zap.SugaredLogger, - clusterId string, - ruleId *string, - toApply []rulefmt.RuleGroup, -) error { - var errArr []error - for _, rules := range toApply { - err := applyCortexSLORules( - ctx, - p, - lg, - clusterId, - rules, - ) - if err != nil { - errArr = append(errArr, err) - } - } - if len(errArr) > 0 { - for _, rules := range toApply { - err := deleteCortexSLORules( - ctx, - p, - lg, - clusterId, - rules.Name, - ) - if err != nil { - errArr = append(errArr, err) - } - } - } - if ruleId != nil { - err := createGrafanaSLOMask(ctx, p, clusterId, *ruleId) - if err != nil { - lg.Errorf("creating grafana mask failed %s", err) - errArr = append(errArr, err) - } - } - - return errors.Combine(errArr...) -} - -// Apply Cortex Rules to Cortex separately : -// - recording rules -// - metadata rules -// - alert rules -func applyCortexSLORules( - ctx context.Context, - p *Plugin, - lg *zap.SugaredLogger, - clusterId string, - ruleSpec rulefmt.RuleGroup, -) error { - out, err := yaml.Marshal(ruleSpec) - if err != nil { - return err - } - - _, err = p.adminClient.Get().LoadRules(ctx, &cortexadmin.LoadRuleRequest{ - Namespace: "slo", - YamlContent: out, - ClusterId: clusterId, - }) - if err != nil { - lg.Error(fmt.Sprintf( - "Failed to load rules for cluster %s, rule : %s,", - clusterId, string(out))) - } - return err -} - -// } -func deleteCortexSLORules( - ctx context.Context, - p *Plugin, - _ *zap.SugaredLogger, - clusterId string, - groupName string, -) error { - _, err := p.adminClient.Get().DeleteRule(ctx, &cortexadmin.DeleteRuleRequest{ - ClusterId: clusterId, - Namespace: "slo", - GroupName: groupName, - }) - // we can ignore 404s here since if we can't find them, - // then it will be impossible to delete them anyway - if status.Code(err) == codes.NotFound || status.Code(err) == codes.OK { - return nil - } - return err -} - -func QuerySLOComponentByRecordName( - ctx context.Context, - client cortexadmin.CortexAdminClient, - recordName string, - clusterId string, -) (*model.Vector, error) { - resp, err := client.Query(ctx, &cortexadmin.QueryRequest{ - Tenants: []string{clusterId}, - Query: recordName, - }) - if err != nil { - return nil, err - } - rawBytes := resp.Data - qres, err := compat.UnmarshalPrometheusResponse(rawBytes) - if err != nil { - return nil, err - } - dataVector, err := qres.GetVector() - if err != nil { - return nil, err - } - return dataVector, nil -} - -func QuerySLOComponentByRawQuery( - ctx context.Context, - client cortexadmin.CortexAdminClient, - rawQuery string, - clusterId string, -) (*model.Vector, error) { - resp, err := client.Query(ctx, &cortexadmin.QueryRequest{ - Tenants: []string{clusterId}, - Query: rawQuery, - }) - if err != nil { - return nil, err - } - rawBytes := resp.Data - qres, err := compat.UnmarshalPrometheusResponse(rawBytes) - if err != nil { - return nil, err - } - dataVector, err := qres.GetVector() - if err != nil { - return nil, err - } - return dataVector, nil -} - -func QuerySLOComponentByRawQueryRange( - ctx context.Context, - client cortexadmin.CortexAdminClient, - rawQuery string, - clusterId string, - start time.Time, - end time.Time, - step time.Duration, -) (*model.Matrix, error) { - resp, err := client.QueryRange(ctx, &cortexadmin.QueryRangeRequest{ - Tenants: []string{clusterId}, - Query: rawQuery, - Start: timestamppb.New(start), - End: timestamppb.New(end), - Step: durationpb.New(step), - }) - if err != nil { - return nil, err - } - rawBytes := resp.Data - qres, err := compat.UnmarshalPrometheusResponse(rawBytes) - if err != nil { - return nil, err - } - dataMatrix, err := qres.GetMatrix() - if err != nil { - return nil, err - } - return dataMatrix, nil -} diff --git a/plugins/slo/pkg/slo/filters.go b/plugins/slo/pkg/slo/filters.go deleted file mode 100644 index 9b1706622f..0000000000 --- a/plugins/slo/pkg/slo/filters.go +++ /dev/null @@ -1,165 +0,0 @@ -package slo - -import ( - "embed" - "fmt" - "io/fs" - "path/filepath" - "regexp" - - "github.com/rancher/opni/pkg/logger" - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - sloapi "github.com/rancher/opni/plugins/slo/apis/slo" - "go.uber.org/zap" - "gopkg.in/yaml.v3" -) - -//go:embed metricgroups/*.yaml -var MetricGroups embed.FS - -//go:embed servicegroups/*.yaml -var ServiceGroups embed.FS - -// map of directory names to their embed.FS -var EnabledFilters = map[string]embed.FS{"metricgroups": MetricGroups, "servicegroups": ServiceGroups} -var filters = constructFilters(logger.NewPluginLogger().Named("slo")) - -// Regexp adds unmarshalling from json for regexp.Regexp -type Regexp struct { - *regexp.Regexp -} - -// UnmarshalText unmarshals json into a regexp.Regexp -func (r *Regexp) UnmarshalText(b []byte) error { - regex, err := regexp.Compile(string(b)) - if err != nil { - return err - } - - r.Regexp = regex - - return nil -} - -// MarshalText marshals regexp.Regexp as string -func (r *Regexp) MarshalText() ([]byte, error) { - if r.Regexp != nil { - return []byte(r.Regexp.String()), nil - } - - return nil, nil -} - -type Filter struct { - Name string `yaml:"name"` - Filters []FilterValue `yaml:"filters"` - Ignore []FilterValue `yaml:"ignore"` -} - -type FilterValue struct { - Value Regexp `yaml:"value"` - Score int `yaml:"score"` -} - -func GetGroupConfigsFromEmbed(lg *zap.SugaredLogger, dirName string, dir embed.FS) []Filter { - lg = lg.With("plugin", "slo", "phase", "init") - var res []Filter - fsys := fs.FS(dir) - yamlFs, err := fs.Sub(fsys, dirName) - if err != nil { - lg.Error(err) - lg.Debug(fmt.Sprintf("Debug error : yamlFs is %s", yamlFs)) - } - err = fs.WalkDir(yamlFs, ".", func(pathStr string, d fs.DirEntry, err error) error { - if !d.IsDir() { - fBytes, err := fs.ReadFile(dir, filepath.Join(dirName, d.Name())) - if err != nil { - lg.Error(err) - lg.Debug(fmt.Sprintf("Debug error : yamlFs is %s", yamlFs)) - panic(err) - } - var f Filter - err = yaml.Unmarshal(fBytes, &f) - if err != nil { - lg.Error(err) - lg.Debug(fmt.Sprintf("Debug error : yamlFs is %s", yamlFs)) - } - res = append(res, f) - } - return nil - }) - if err != nil { - lg.Error(err) - lg.Debug(fmt.Sprintf("Debug error : yamlFs is %s", yamlFs)) - } - return res -} - -func constructFilters(lg *zap.SugaredLogger) []Filter { - res := []Filter{} - for dirName, embedFs := range EnabledFilters { - filters := GetGroupConfigsFromEmbed(lg, dirName, embedFs) - res = append(res, filters...) - } - return res -} - -// Map metric -> group -> score -func scoredLabels(seriesInfo *cortexadmin.SeriesInfoList) *sloapi.MetricGroupList { - res := map[*cortexadmin.SeriesInfo]map[string]int{} - - for _, series := range seriesInfo.GetItems() { - series.GetSeriesName() - for _, groupname := range filters { - for _, matchFilter := range groupname.Filters { - if matchFilter.Value.MatchString(series.GetSeriesName()) { - if _, ok := res[series]; !ok { - res[series] = map[string]int{} - } - res[series][groupname.Name] += matchFilter.Score - } - } - - for _, ignoreFilter := range groupname.Ignore { - if ignoreFilter.Value.MatchString(series.GetSeriesName()) { - if _, ok := res[series]; !ok { - res[series] = map[string]int{} - } - res[series][groupname.Name] -= ignoreFilter.Score - } - } - } - } - groupedMetrics := sloapi.MetricGroupList{ - GroupNameToMetrics: map[string]*sloapi.MetricList{}, - } - for series, vals := range res { - groupName := "" - maxScore := 0 - for group, score := range vals { - if score > maxScore { - maxScore = score - groupName = group - } - } - if maxScore <= 0 { - groupName = "other metrics" - } - if _, ok := groupedMetrics.GroupNameToMetrics[groupName]; !ok { - groupedMetrics.GroupNameToMetrics[groupName] = &sloapi.MetricList{} - } - groupedMetrics.GroupNameToMetrics[groupName].Items = append(groupedMetrics.GroupNameToMetrics[groupName].Items, &sloapi.Metric{ - Id: series.GetSeriesName(), - Metadata: &sloapi.MetricMetadata{ - Description: series.Metadata.GetDescription(), - Unit: series.Metadata.GetUnit(), - Type: series.Metadata.GetType(), - }, - }) - } - return &groupedMetrics -} - -func ApplyFiltersToCortexEvents(seriesInfo *cortexadmin.SeriesInfoList) (*sloapi.MetricGroupList, error) { - return scoredLabels(seriesInfo), nil -} diff --git a/plugins/slo/pkg/slo/filters_test.go b/plugins/slo/pkg/slo/filters_test.go deleted file mode 100644 index 1b072abc3f..0000000000 --- a/plugins/slo/pkg/slo/filters_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package slo_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/rancher/opni/pkg/logger" - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - "github.com/rancher/opni/plugins/slo/pkg/slo" -) - -var _ = Describe("SLO Filter tests", Ordered, Label("unit", "slow"), func() { - { - When("We use SLO filters", func() { - It("should get parse them from our embedded directory definitions", func() { - for dirName, embedFs := range slo.EnabledFilters { - filters := slo.GetGroupConfigsFromEmbed(logger.NewPluginLogger().Named("slo"), dirName, embedFs) - Expect(filters).NotTo(HaveLen(0)) - for _, filter := range filters { - Expect(filter.Name).NotTo(Equal("")) - Expect(filter.Filters).NotTo(BeEmpty()) - } - } - }) - - It("Should be able to score the events based on the filters", func() { - // array of prom metrics : label name -> label vals - series := &cortexadmin.SeriesInfoList{ - Items: []*cortexadmin.SeriesInfo{ - { - SeriesName: "go_gc_duration_seconds", - }, - { - SeriesName: "jvm_something", - }, - { - SeriesName: "jvm_something_else", - }, - { - SeriesName: "uptime_seconds", - }, - { - SeriesName: "kube-proxy", - }, - { - SeriesName: "apiserver_something", - }, - { - SeriesName: "request_duration_seconds", - }, - { - SeriesName: "cpu_usage_seconds_total", - }, - }, - } - - filteredGroups, err := slo.ApplyFiltersToCortexEvents(series) - Expect(err).To(Succeed()) - Expect(filteredGroups).NotTo(BeNil()) - Expect(filteredGroups.GroupNameToMetrics["golang metrics"].Items).To(HaveLen(1)) - Expect(filteredGroups.GroupNameToMetrics["jvm metrics"].Items).To(HaveLen(2)) - Expect(filteredGroups.GroupNameToMetrics["kubernetes metrics"].Items).To(HaveLen(1)) - Expect(filteredGroups.GroupNameToMetrics["network metrics"].Items).To(HaveLen(1)) - Expect(filteredGroups.GroupNameToMetrics["compute metrics"].Items).To(HaveLen(1)) - Expect(filteredGroups.GroupNameToMetrics["other metrics"].Items).To(HaveLen(2)) - }) - }) - } -}) diff --git a/plugins/slo/pkg/slo/impl.go b/plugins/slo/pkg/slo/impl.go deleted file mode 100644 index e21d132a73..0000000000 --- a/plugins/slo/pkg/slo/impl.go +++ /dev/null @@ -1,422 +0,0 @@ -package slo - -import ( - "context" - "fmt" - "strconv" - "sync" - "time" - - "slices" - - "emperror.dev/errors" - "github.com/google/uuid" - prommodel "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/rulefmt" - "github.com/prometheus/prometheus/promql/parser" - corev1 "github.com/rancher/opni/pkg/apis/core/v1" - managementv1 "github.com/rancher/opni/pkg/apis/management/v1" - "github.com/rancher/opni/pkg/util" - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - sloapi "github.com/rancher/opni/plugins/slo/apis/slo" - "github.com/samber/lo" - "github.com/tidwall/gjson" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/timestamppb" -) - -func (s *SLOMonitoring) WithCurrentRequest(ctx context.Context, req proto.Message) SLOStore { - s.req = req - s.ctx = ctx - return s -} - -func (s SLOMonitoring) Create() (*corev1.Reference, error) { - req := (s.req).(*sloapi.CreateSLORequest) - slo := CreateSLORequestToStruct(req) - rrecording, rmetadata, ralerting := slo.ConstructCortexRules(nil) - toApply := []rulefmt.RuleGroup{rrecording, rmetadata, ralerting} - ruleId := slo.GetId() - err := tryApplyThenDeleteCortexRules(s.ctx, s.p, s.p.logger, req.GetSlo().GetClusterId(), &ruleId, toApply) - if err != nil { - return nil, err - } - return &corev1.Reference{Id: slo.GetId()}, nil -} - -func (s SLOMonitoring) Update(existing *sloapi.SLOData) (*sloapi.SLOData, error) { - incomingSLO := (s.req).(*sloapi.SLOData) // Create is the same as Update if within the same cluster - newSlo := SLODataToStruct(incomingSLO) - rrecording, rmetadata, ralerting := newSlo.ConstructCortexRules(nil) - toApply := []rulefmt.RuleGroup{rrecording, rmetadata, ralerting} - err := tryApplyThenDeleteCortexRules(s.ctx, s.p, s.p.logger, incomingSLO.GetSLO().GetClusterId(), nil, toApply) - - // successfully applied rules to another cluster - if err == nil && existing.SLO.ClusterId != incomingSLO.SLO.ClusterId { - _, err := s.p.DeleteSLO(s.ctx, &corev1.Reference{Id: existing.Id}) - if err != nil { - s.lg.With("sloId", existing.Id).Error(fmt.Sprintf( - "Unable to delete SLO when updating between clusters : %v", - err)) - } - } - return incomingSLO, err -} - -func (s SLOMonitoring) Delete(existing *sloapi.SLOData) error { - id, clusterId := existing.Id, existing.SLO.ClusterId - //err := deleteCortexSLORules(s.p, id, clusterId, s.ctx, s.lg) - errArr := []error{} - slo := SLODataToStruct(existing) - rrecording, rmetadata, ralerting := slo.ConstructCortexRules(nil) - toApply := []rulefmt.RuleGroup{rrecording, rmetadata, ralerting} - for _, ruleName := range toApply { - for _, rule := range ruleName.Rules { - if rule.Alert.Value != "" { - err := deleteCortexSLORules( - s.ctx, - s.p, - s.p.logger, - clusterId, - rule.Alert.Value, - ) - if err != nil { - errArr = append(errArr, err) - } - } - if rule.Record.Value != "" { - err := deleteCortexSLORules( - s.ctx, - s.p, - s.p.logger, - clusterId, - rule.Record.Value, - ) - if err != nil { - errArr = append(errArr, err) - } - } - } - } - err := createGrafanaSLOMask(s.ctx, s.p, clusterId, id) - if err != nil { - s.p.logger.Errorf("creating grafana mask failed %s", err) - errArr = append(errArr, err) - } - return errors.Combine(errArr...) -} - -func (s SLOMonitoring) Clone(clone *sloapi.SLOData) (*corev1.Reference, *sloapi.SLOData, error) { - clonedData := util.ProtoClone(clone) - sloData := clonedData.GetSLO() - slo := SLODataToStruct(clonedData) - slo.SetId(uuid.New().String()) - slo.SetName(sloData.GetName() + "-clone") - rrecording, rmetadata, ralerting := slo.ConstructCortexRules(nil) - toApply := []rulefmt.RuleGroup{rrecording, rmetadata, ralerting} - ruleId := slo.GetId() - err := tryApplyThenDeleteCortexRules(s.ctx, s.p, s.p.logger, sloData.GetClusterId(), &ruleId, toApply) - clonedData.SLO.Name = sloData.Name + "-clone" - clonedData.Id = slo.GetId() - return &corev1.Reference{Id: slo.GetId()}, clonedData, err -} - -func (s SLOMonitoring) MultiClusterClone( - base *sloapi.SLOData, - inputClusters []*corev1.Reference, - svcBackend ServiceBackend, -) ([]*corev1.Reference, []*sloapi.SLOData, []error) { - clonedData := util.ProtoClone(base) - sloData := clonedData.GetSLO() - slo := SLODataToStruct(clonedData) - - clusters, err := s.p.mgmtClient.Get().ListClusters(s.ctx, &managementv1.ListClustersRequest{}) - if err != nil { - return nil, nil, []error{err} - } - var clusterIds []string - - for _, cluster := range clusters.Items { - clusterIds = append(clusterIds, cluster.Id) - } - clusterDefinitions := make([]*sloapi.SLOData, len(inputClusters)) - clusterIdsCreate := make([]*corev1.Reference, len(inputClusters)) - errArr := make([]error, len(inputClusters)) - var wg sync.WaitGroup - for idx, clusterId := range inputClusters { - wg.Add(1) - slo.SetId(uuid.New().String()) - slo.SetName(fmt.Sprintf("%s-clone-%d", sloData.GetName(), idx)) - rrecording, rmetadata, ralerting := slo.ConstructCortexRules(nil) - toApply := []rulefmt.RuleGroup{rrecording, rmetadata, ralerting} - // capture in closure - idx := idx - clusterId := clusterId - ruleId := slo.GetId() - if !slices.Contains(clusterIds, clusterId.Id) { - errArr[idx] = fmt.Errorf("cluster %s not found", clusterId.Id) - continue - } - svcBackend.WithCurrentRequest(s.ctx, &sloapi.ListServicesRequest{ - Datasource: "monitoring", - ClusterId: clusterId.Id, - }) - services, err := svcBackend.ListServices() - if err != nil { - errArr[idx] = err - continue - } - if services.ContainsId(sloData.GetServiceId()) { - errArr[idx] = fmt.Errorf("service %s not found on cluster %s", sloData.GetServiceId(), clusterId.Id) - continue - } - svcBackend.WithCurrentRequest(s.ctx, &sloapi.ListMetricsRequest{ - Datasource: "monitoring", - ClusterId: clusterId.Id, - ServiceId: sloData.GetServiceId(), - }) - metrics, err := svcBackend.ListMetrics() - if err != nil { - errArr[idx] = err - continue - } - if !metrics.ContainsId(sloData.GetGoodMetricName()) { - errArr[idx] = fmt.Errorf( - "good metric %s not found on cluster %s", - sloData.GetGoodMetricName(), - clusterId.Id, - ) - continue - } - if !metrics.ContainsId(sloData.GetTotalMetricName()) { - errArr[idx] = fmt.Errorf( - "total metric %s not found on cluster %s", - sloData.GetTotalMetricName(), - clusterId.Id, - ) - continue - } - errArr[idx] = tryApplyThenDeleteCortexRules(s.ctx, s.p, s.p.logger, clusterId.Id, &ruleId, toApply) - clonedData.SLO.Name = sloData.Name + "-clone-" + strconv.Itoa(idx) - clonedData.Id = slo.GetId() - clusterDefinitions[idx] = clonedData - clusterIdsCreate[idx] = &corev1.Reference{Id: slo.GetId()} - } - return clusterIdsCreate, clusterDefinitions, errArr -} - -// Status Only return errors here that should be considered severe InternalServerErrors -// - Check if enough time has passed to evaluate the rules -// - First Checks if it has NoData -// - If it has Data, check if it is within budget -// - If is within budget, check if any alerts are firing -func (s SLOMonitoring) Status(existing *sloapi.SLOData) (*sloapi.SLOStatus, error) { - now := time.Now() - - if now.Sub(existing.CreatedAt.AsTime()) <= sloapi.MinEvaluateInterval*2 { - s.lg.With("sloId", existing.Id).Debug("SLO status is not ready to be evaluated : ", - (&sloapi.SLOStatus{State: sloapi.SLOStatusState_Creating}).String()) - - return &sloapi.SLOStatus{State: sloapi.SLOStatusState_Creating}, nil - } - state := sloapi.SLOStatusState_Ok - slo := SLODataToStruct(existing) - // ======================= sli ======================= - sliErrorName := slo.ConstructRecordingRuleGroup(nil).Rules[0].Record - sliDataVector, err := QuerySLOComponentByRecordName( - s.ctx, - s.p.adminClient.Get(), - sliErrorName.Value, - existing.GetSLO().GetClusterId(), - ) - if err != nil { - return nil, err - } - if sliDataVector == nil || sliDataVector.Len() == 0 { - return &sloapi.SLOStatus{State: sloapi.SLOStatusState_NoData}, nil - } - s.lg.With("sloId", slo.GetId()).Debug("sli status response vector : ", sliDataVector.String()) - // ======================= error budget ======================= - // race condition can cause initial evaluation to fail with empty vector, resulting in no data state - // this is why we return creating state with two intervals - metadataBudgetRaw := slo.RawBudgetRemainingQuery() // this is not actually raw "raw", contains recording rule refs - metadataVector, err := QuerySLOComponentByRawQuery(s.ctx, s.p.adminClient.Get(), metadataBudgetRaw, existing.GetSLO().GetClusterId()) - if err != nil { - return nil, err - } - if metadataVector == nil || metadataVector.Len() == 0 { - return &sloapi.SLOStatus{State: sloapi.SLOStatusState_PartialDataOk}, nil - } - metadataBudget := (*metadataVector)[0].Value - if metadataBudget <= 0 { - return &sloapi.SLOStatus{State: sloapi.SLOStatusState_Breaching}, nil - } - s.lg.With("sloId", slo.GetId()).Debug("sli status ", metadataVector.String()) - // - //// ======================= alert ======================= - - alertBudgetRules := slo.ConstructAlertingRuleGroup(nil) - short, long := alertBudgetRules.Rules[0].Expr.Value, alertBudgetRules.Rules[1].Expr.Value - alertDataVector1, err := QuerySLOComponentByRawQuery(s.ctx, s.p.adminClient.Get(), short, existing.GetSLO().GetClusterId()) - if err != nil { - return nil, err - } - alertDataVector2, err := QuerySLOComponentByRawQuery(s.ctx, s.p.adminClient.Get(), long, existing.GetSLO().GetClusterId()) - if err != nil { - return nil, err - } - if alertDataVector1 == nil || alertDataVector1.Len() == 0 || alertDataVector2 == nil || alertDataVector2.Len() == 0 { - return &sloapi.SLOStatus{State: sloapi.SLOStatusState_PartialDataOk}, nil - } - if (*alertDataVector1)[len(*alertDataVector1)-1].Value > 0 || (*alertDataVector2)[len(*alertDataVector2)-1].Value > 0 { - return &sloapi.SLOStatus{State: sloapi.SLOStatusState_Warning}, nil - } - s.lg.With("sloId", slo.GetId()).Debug("alert status response vector ", alertDataVector1.String(), alertDataVector2.String()) - return &sloapi.SLOStatus{ - State: state, - }, nil -} - -func (s SLOMonitoring) Preview(slo *SLO) (*sloapi.SLOPreviewResponse, error) { - req := s.req.(*sloapi.CreateSLORequest) - preview := &sloapi.SLOPreviewResponse{ - PlotVector: &sloapi.PlotVector{ - Objective: normalizeObjective(req.GetSlo().GetTarget().GetValue()), - Items: []*sloapi.DataPoint{}, - Windows: []*sloapi.AlertFiringWindows{}, - }, - } - cur := time.Now() - dur, err := prommodel.ParseDuration(slo.sloPeriod) - if err != nil { - panic(err) - } - startTs, endTs := cur.Add(time.Duration(-dur)), cur - numSteps := 250 - step := time.Duration(endTs.Sub(startTs).Seconds()/float64(numSteps)) * time.Second - if step < time.Second { - step = time.Second - } - - ruleGroup := slo.ConstructRecordingRuleGroup(lo.ToPtr(sloapi.MinEvaluateInterval)) - sliPeriodErrorRate := ruleGroup.Rules[len(ruleGroup.Rules)-1].Expr.Value - sli := "1 - (max(" + sliPeriodErrorRate + ") OR on() vector(NaN))" // handles the empty case and still differentiates between 0 and empty - _, err = parser.ParseExpr(sli) - if err != nil { - panic(err) - } - sliDataMatrix, err := QuerySLOComponentByRawQueryRange(s.ctx, s.p.adminClient.Get(), - sli, req.GetSlo().GetClusterId(), - startTs, endTs, step, - ) - if err != nil { - return nil, err - } - for _, sample := range *sliDataMatrix { - for _, yieldedValue := range sample.Values { - ts := time.Unix(yieldedValue.Timestamp.Unix(), 0) - preview.PlotVector.Items = append(preview.PlotVector.Items, &sloapi.DataPoint{ - Timestamp: timestamppb.New(ts), - Sli: float64(yieldedValue.Value) * 100, - }) - } - } - - alertCriticalRawQuery, alertSevereRawQuery := slo.ConstructRawAlertQueries() - // ideally should be every 5 minutes for fine grained detail - // but for performance reasons, we will only query every 20 minutes - alertTimeStep := time.Minute * 20 - - alertWindowSevereMatrix, err := QuerySLOComponentByRawQueryRange(s.ctx, s.p.adminClient.Get(), - alertSevereRawQuery.Value, req.GetSlo().GetClusterId(), - startTs, endTs, alertTimeStep, - ) - if err != nil { - return nil, err - } - severeWindows, err := DetectActiveWindows("severe", alertWindowSevereMatrix) - if err != nil { - return nil, err - } - preview.PlotVector.Windows = append(preview.PlotVector.Windows, severeWindows...) - - alertWindowCriticalMatrix, err := QuerySLOComponentByRawQueryRange(s.ctx, s.p.adminClient.Get(), - alertCriticalRawQuery.Value, req.GetSlo().GetClusterId(), - startTs, endTs, step, - ) - if err != nil { - return nil, err - } - criticalWindows, err := DetectActiveWindows("critical", alertWindowCriticalMatrix) - if err != nil { - return nil, err - } - preview.PlotVector.Windows = append(preview.PlotVector.Windows, criticalWindows...) - return preview, nil -} - -func (m *MonitoringServiceBackend) WithCurrentRequest(ctx context.Context, req proto.Message) ServiceBackend { - m.req = req - m.ctx = ctx - return m -} - -func (m MonitoringServiceBackend) ListServices() (*sloapi.ServiceList, error) { - req := m.req.(*sloapi.ListServicesRequest) - res := &sloapi.ServiceList{} - discoveryQuery := `group by(job) ({__name__!=""})` - resp, err := m.p.adminClient.Get().Query( - m.ctx, - &cortexadmin.QueryRequest{ - Tenants: []string{req.GetClusterId()}, - Query: discoveryQuery, - }) - if err != nil { - return nil, err - } - result := gjson.Get(string(resp.Data), "data.result.#.metric.job") - if !result.Exists() { - return nil, fmt.Errorf("Could not convert prometheus service discovery to json ") - } - for _, v := range result.Array() { - res.Items = append(res.Items, &sloapi.Service{ - ClusterId: req.GetClusterId(), - ServiceId: v.String(), - }) - } - return res, nil -} - -func (m MonitoringServiceBackend) ListEvents() (*sloapi.EventList, error) { - res := &sloapi.EventList{ - Items: []*sloapi.Event{}, - } - req := (m.req).(*sloapi.ListEventsRequest) // Create is the same as Update if within the same cluster - resp, err := m.p.adminClient.Get().GetMetricLabelSets(m.ctx, &cortexadmin.LabelRequest{ - Tenant: req.GetClusterId(), - JobId: req.GetServiceId(), - MetricName: req.GetMetricId(), - }) - if err != nil { - return nil, err - } - for _, item := range resp.GetItems() { - res.Items = append(res.Items, &sloapi.Event{ - Key: item.GetName(), - Vals: item.GetItems(), - }) - } - return res, nil -} - -func (m MonitoringServiceBackend) ListMetrics() (*sloapi.MetricGroupList, error) { - req := (m.req).(*sloapi.ListMetricsRequest) // Create is the same as Update if within the same cluster - resp, err := m.p.adminClient.Get().GetSeriesMetrics(m.ctx, &cortexadmin.SeriesRequest{ - Tenant: req.GetClusterId(), - JobId: req.GetServiceId(), - }) - if err != nil { - return nil, err - } - return ApplyFiltersToCortexEvents(resp) -} diff --git a/plugins/slo/pkg/slo/interfaces.go b/plugins/slo/pkg/slo/interfaces.go deleted file mode 100644 index 7256108bc9..0000000000 --- a/plugins/slo/pkg/slo/interfaces.go +++ /dev/null @@ -1,92 +0,0 @@ -package slo - -import ( - "context" - "sync" - - corev1 "github.com/rancher/opni/pkg/apis/core/v1" - sloapi "github.com/rancher/opni/plugins/slo/apis/slo" - "go.uber.org/zap" - - "google.golang.org/protobuf/proto" -) - -var datasourceToSLO = make(map[string]SLOStore) -var datasourceToService = make(map[string]ServiceBackend) -var mu sync.Mutex - -func RegisterDatasource(datasource string, sloImpl SLOStore, serviceImpl ServiceBackend) { - defer mu.Unlock() - mu.Lock() - datasourceToSLO[datasource] = sloImpl - datasourceToService[datasource] = serviceImpl -} - -type SLOStore interface { - // This method has to handle storage of the SLO in the KVStore itself - // since there can be partial successes inside the method - Create() (*corev1.Reference, error) - Update(existing *sloapi.SLOData) (*sloapi.SLOData, error) - Delete(existing *sloapi.SLOData) error - Clone(clone *sloapi.SLOData) (*corev1.Reference, *sloapi.SLOData, error) - MultiClusterClone( - base *sloapi.SLOData, - clusters []*corev1.Reference, - svcBackend ServiceBackend, - ) ([]*corev1.Reference, []*sloapi.SLOData, []error) - Status(existing *sloapi.SLOData) (*sloapi.SLOStatus, error) - Preview(s *SLO) (*sloapi.SLOPreviewResponse, error) - WithCurrentRequest(ctx context.Context, req proto.Message) SLOStore -} -type ServiceBackend interface { - ListServices() (*sloapi.ServiceList, error) - ListMetrics() (*sloapi.MetricGroupList, error) - ListEvents() (*sloapi.EventList, error) - WithCurrentRequest(ctx context.Context, req proto.Message) ServiceBackend -} - -type MetricIds struct { - Good string - Total string -} - -type RequestBase struct { - req proto.Message - p *Plugin - ctx context.Context - lg *zap.SugaredLogger -} - -type SLOMonitoring struct { - RequestBase -} - -type SLOLogging struct { - RequestBase -} - -type MonitoringServiceBackend struct { - RequestBase -} - -func NewSLOMonitoringStore(p *Plugin, lg *zap.SugaredLogger) SLOStore { - return &SLOMonitoring{ - RequestBase{ - req: nil, - p: p, - ctx: context.Background(), - lg: lg, - }, - } -} - -func NewMonitoringServiceBackend(p *Plugin, lg *zap.SugaredLogger) ServiceBackend { - return &MonitoringServiceBackend{ - RequestBase{ - req: nil, - p: p, - ctx: context.TODO(), - lg: lg, - }, - } -} diff --git a/plugins/slo/pkg/slo/plugin.go b/plugins/slo/pkg/slo/plugin.go deleted file mode 100644 index 00bf7c3186..0000000000 --- a/plugins/slo/pkg/slo/plugin.go +++ /dev/null @@ -1,60 +0,0 @@ -package slo - -import ( - "context" - - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - "github.com/rancher/opni/plugins/slo/apis/slo" - "go.uber.org/zap" - - alertingv1 "github.com/rancher/opni/pkg/apis/alerting/v1" - managementv1 "github.com/rancher/opni/pkg/apis/management/v1" - "github.com/rancher/opni/pkg/logger" - managementext "github.com/rancher/opni/pkg/plugins/apis/apiextensions/management" - "github.com/rancher/opni/pkg/plugins/apis/system" - "github.com/rancher/opni/pkg/plugins/meta" - "github.com/rancher/opni/pkg/storage" - "github.com/rancher/opni/pkg/util" - "github.com/rancher/opni/pkg/util/future" -) - -type Plugin struct { - slo.UnsafeSLOServer - system.UnimplementedSystemPluginClient - - ctx context.Context - logger *zap.SugaredLogger - - storage future.Future[StorageAPIs] - mgmtClient future.Future[managementv1.ManagementClient] - adminClient future.Future[cortexadmin.CortexAdminClient] - alertEndpointClient future.Future[alertingv1.AlertEndpointsClient] -} - -type StorageAPIs struct { - SLOs storage.KeyValueStoreT[*slo.SLOData] - Services storage.KeyValueStoreT[*slo.Service] - Metrics storage.KeyValueStoreT[*slo.Metric] -} - -func NewPlugin(ctx context.Context) *Plugin { - return &Plugin{ - ctx: ctx, - logger: logger.NewPluginLogger().Named("slo"), - storage: future.New[StorageAPIs](), - mgmtClient: future.New[managementv1.ManagementClient](), - adminClient: future.New[cortexadmin.CortexAdminClient](), - alertEndpointClient: future.New[alertingv1.AlertEndpointsClient](), - } -} - -var _ slo.SLOServer = (*Plugin)(nil) - -func Scheme(ctx context.Context) meta.Scheme { - scheme := meta.NewScheme() - p := NewPlugin(ctx) - scheme.Add(system.SystemPluginID, system.NewPlugin(p)) - scheme.Add(managementext.ManagementAPIExtensionPluginID, - managementext.NewPlugin(util.PackService(&slo.SLO_ServiceDesc, p))) - return scheme -} diff --git a/plugins/slo/pkg/slo/simple.go b/plugins/slo/pkg/slo/simple.go deleted file mode 100644 index 7980035de0..0000000000 --- a/plugins/slo/pkg/slo/simple.go +++ /dev/null @@ -1,859 +0,0 @@ -package slo - -import ( - "bytes" - "fmt" - "os" - "strings" - "text/template" - "time" - - "github.com/google/uuid" - prommodel "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/rulefmt" - "github.com/prometheus/prometheus/promql/parser" - "github.com/rancher/opni/pkg/alerting/metrics" - sloapi "github.com/rancher/opni/plugins/slo/apis/slo" - "github.com/samber/lo" - "google.golang.org/protobuf/types/known/timestamppb" - "gopkg.in/yaml.v3" -) - -const ( - grafana_delete_mask = "slo_opni_delete" - // id labels - slo_uuid = "slo_opni_id" - slo_service = "slo_opni_service" - slo_name = "slo_opni_name" - slo_window = "slo_window" - - // recording rule names - slo_ratio_rate_query_name = "slo:sli_error:ratio_rate" - slo_alert_ticket_window = "slo:alert:ticket_window" - slo_alert_page_window = "slo:alert:page_window" - - // metadata rule names - slo_objective_ratio = "slo:objective:ratio" - slo_error_budget_ratio = "slo:error_budget:ratio" - slo_time_period_days = "slo:time_period:days" - slo_current_burn_rate_ratio = "slo:current_burn_rate:ratio" - slo_period_burn_rate_ratio = "slo:period_burn_rate:ratio" - slo_period_error_budget_remaining_ratio = "slo:period_error_budget_remaining:ratio" - slo_info = "opni_slo_info" - - // alert rule names - RecordingRuleSuffix = "-recording" - MetadataRuleSuffix = "-metadata" - AlertRuleSuffix = "-alerts" -) - -var ( - simpleQueryTpl = template.Must(template.New("query").Parse(`sum(rate({{.Metric}}{job="{{.JobId}}"{{.Labels}}}[{{.Window}}]))`)) - rawSliQueryTpl = template.Must(template.New("sliRawQuery").Parse(`1 - (({{.GoodQuery}})/({{.TotalQuery}}))`)) - sloFiltersTpl = template.Must(template.New("sloFilters").Parse(`{{.SloIdLabel}}="{{.SloId}}", {{.SloServiceLabel}}="{{.SloService}}", {{.SloNameLabel}}="{{.SloName}}"`)) -) - -// SliQueryInfo used for filling sli query templates -type SliQueryInfo struct { - GoodQuery string - TotalQuery string -} - -type SloFiltersInfo struct { - SloIdLabel string - SloServiceLabel string - SloNameLabel string - SloId string - SloService string - SloName string -} - -//type RuleGroupYAMLv2 struct { -// Name string `yaml:"name"` -// Interval prommodel.Duration `yaml:"interval,omitempty"` -// Rules []rulefmt.Rule `yaml:"rules"` -//} - -type LabelPair struct { - Key string - Vals []string -} - -type LabelPairs []LabelPair - -func (l LabelPairs) Construct() string { // kinda hacky & technically unoptimized but works - if len(l) == 0 { - return "" - } - s := "" - for _, labelPair := range l { - if labelPair.Key == "" { - continue - } - orCompositionVal := "" - if len(labelPair.Vals) == 1 { - orCompositionVal += labelPair.Vals[0] - } else { - for idx, val := range labelPair.Vals { - orCompositionVal += val - if idx != len(labelPair.Vals)-1 { - orCompositionVal += "|" - } - } - } - s += fmt.Sprintf(",%s=~\"%s\"", labelPair.Key, orCompositionVal) - } - return s -} - -type IdentificationLabels map[string]string -type UserLabels []string -type Service string -type Metric string - -type SLO struct { - sloPeriod string - objective float64 // 0 < x < 100 - svc Service - goodMetric Metric - totalMetric Metric - idLabels IdentificationLabels - userLabels map[string]string - goodEvents LabelPairs - totalEvents LabelPairs -} - -func normalizeObjective(objective float64) float64 { - return objective / 100 -} - -func NewSLO( - sloName string, - sloPeriod string, - objective float64, - svc Service, - goodMetric Metric, - totalMetric Metric, - userLabels map[string]string, - goodEvents []LabelPair, - totalEvents []LabelPair, -) *SLO { - newId := uuid.New().String() - ilabels := IdentificationLabels{slo_uuid: newId, slo_name: sloName, slo_service: string(svc)} - - return &SLO{ - svc: svc, - sloPeriod: sloPeriod, - goodMetric: goodMetric, - totalMetric: totalMetric, - userLabels: userLabels, - goodEvents: goodEvents, - totalEvents: totalEvents, - idLabels: ilabels, - objective: objective, - } -} - -func SLOFromId( - sloName string, - sloPeriod string, - objective float64, - svc Service, - goodMetric Metric, - totalMetric Metric, - userLabels map[string]string, - goodEvents []LabelPair, - totalEvents []LabelPair, - id string, -) *SLO { - ilabels := IdentificationLabels{slo_uuid: id, slo_name: sloName, slo_service: string(svc)} - - return &SLO{ - svc: svc, - goodMetric: goodMetric, - totalMetric: totalMetric, - sloPeriod: sloPeriod, - userLabels: userLabels, - goodEvents: goodEvents, - totalEvents: totalEvents, - idLabels: ilabels, - objective: objective, - } -} - -func CreateSLORequestToStruct(c *sloapi.CreateSLORequest) *SLO { - if c.Slo.GetGoodMetricName() == c.Slo.GetTotalMetricName() { - c.Slo.GoodEvents, c.Slo.TotalEvents = ToMatchingSubsetIdenticalMetric(c.Slo.GoodEvents, c.Slo.TotalEvents) - } - reqSLO := c.Slo - userLabels := reqSLO.GetLabels() - sloLabels := map[string]string{} - for _, label := range userLabels { - sloLabels[label.GetName()] = "true" - } - goodEvents := []LabelPair{} - for _, goodEvent := range reqSLO.GetGoodEvents() { - goodEvents = append(goodEvents, LabelPair{ - Key: goodEvent.GetKey(), - Vals: goodEvent.GetVals(), - }) - } - totalEvents := []LabelPair{} - for _, totalEvent := range reqSLO.GetTotalEvents() { - totalEvents = append(totalEvents, LabelPair{ - Key: totalEvent.GetKey(), - Vals: totalEvent.GetVals(), - }) - } - return NewSLO( - reqSLO.GetName(), - reqSLO.GetSloPeriod(), - reqSLO.GetTarget().GetValue(), - Service(reqSLO.GetServiceId()), - Metric(reqSLO.GetGoodMetricName()), - Metric(reqSLO.GetTotalMetricName()), - sloLabels, - goodEvents, - totalEvents, - ) -} - -func SLODataToStruct(s *sloapi.SLOData) *SLO { - reqSLO := s.SLO - if reqSLO.GetGoodMetricName() == reqSLO.GetTotalMetricName() { - reqSLO.GoodEvents, reqSLO.TotalEvents = ToMatchingSubsetIdenticalMetric(reqSLO.GoodEvents, reqSLO.TotalEvents) - } - userLabels := reqSLO.GetLabels() - sloLabels := map[string]string{} - for _, label := range userLabels { - sloLabels[label.GetName()] = "true" - } - goodEvents := []LabelPair{} - for _, goodEvent := range reqSLO.GetGoodEvents() { - goodEvents = append(goodEvents, LabelPair{ - Key: goodEvent.GetKey(), - Vals: goodEvent.GetVals(), - }) - } - totalEvents := []LabelPair{} - for _, totalEvent := range reqSLO.GetTotalEvents() { - totalEvents = append(totalEvents, LabelPair{ - Key: totalEvent.GetKey(), - Vals: totalEvent.GetVals(), - }) - } - if s.Id == "" { - return NewSLO( - reqSLO.GetName(), - reqSLO.GetSloPeriod(), - reqSLO.GetTarget().GetValue(), - Service(reqSLO.GetServiceId()), - Metric(reqSLO.GetGoodMetricName()), - Metric(reqSLO.GetTotalMetricName()), - sloLabels, - goodEvents, - totalEvents, - ) - } - return SLOFromId( - reqSLO.GetName(), - reqSLO.GetSloPeriod(), - reqSLO.GetTarget().GetValue(), - Service(reqSLO.GetServiceId()), - Metric(reqSLO.GetGoodMetricName()), - Metric(reqSLO.GetTotalMetricName()), - sloLabels, - goodEvents, - totalEvents, - s.Id, - ) -} - -func (s *SLO) GetId() string { - return s.idLabels[slo_uuid] // let it panic if not found -} - -func (s *SLO) SetId(id string) { - s.idLabels[slo_uuid] = id -} - -func (s *SLO) GetName() string { - return s.idLabels[slo_name] // let it panic if not found -} - -func (s *SLO) SetName(input string) { - s.idLabels[slo_name] = input -} - -func (s *SLO) GetPeriod() string { - return s.sloPeriod -} - -func (s *SLO) GetObjective() float64 { - return s.objective -} - -func (s *SLO) GetPrometheusRuleFilterByIdLabels() (string, error) { - var b bytes.Buffer - err := sloFiltersTpl.Execute(&b, SloFiltersInfo{ - SloIdLabel: slo_uuid, - SloServiceLabel: slo_service, - SloNameLabel: slo_name, - SloId: s.GetId(), - SloService: string(s.svc), - SloName: s.GetName(), - }) - return b.String(), err -} - -func (s *SLO) RawObjectiveQuery() string { - return fmt.Sprintf("vector(%.9f)", normalizeObjective(s.objective)) -} - -func (s *SLO) RawErrorBudgetQuery() string { - return fmt.Sprintf("vector(1-%.9f)", normalizeObjective(s.objective)) -} - -// RawCurrentBurnRateQuery -// ratioRate : slo:sli_error:ratio_rate -func (s *SLO) RawCurrentBurnRateQuery() string { - ruleFilters, err := s.GetPrometheusRuleFilterByIdLabels() - if err != nil { - panic(err) - } - fastWindow := NewWindowRange(s.sloPeriod)[0] - strRes := (slo_ratio_rate_query_name + fastWindow) + "{" + ruleFilters + "}" + " / " + - fmt.Sprintf("on(%s, %s, %s)", slo_uuid, slo_service, slo_name) + " group_left\n" + - slo_error_budget_ratio + "{" + ruleFilters + "}" - - return strRes -} - -func (s *SLO) RawPeriodBurnRateQuery() string { - ruleFilters, err := s.GetPrometheusRuleFilterByIdLabels() - if err != nil { - panic(err) - } - slowestWindow := NewWindowRange(s.sloPeriod)[len(NewWindowRange(s.sloPeriod))-1] - strRes := (slo_ratio_rate_query_name + slowestWindow) + "{" + ruleFilters + "}" + " / " + - fmt.Sprintf("on(%s, %s, %s)", slo_uuid, slo_service, slo_name) + " group_left\n" + - slo_error_budget_ratio + "{" + ruleFilters + "}" - return strRes -} - -func (s *SLO) RawPeriodDurationQuery() string { - dur, err := prommodel.ParseDuration(s.sloPeriod) - if err != nil { - panic(err) - } - timeDur := time.Duration(dur) - days := int(timeDur.Hours() / 24) - return fmt.Sprintf("vector(%d)", days) -} - -func (s *SLO) RawBudgetRemainingQuery() string { - ruleFilters, err := s.GetPrometheusRuleFilterByIdLabels() - if err != nil { - panic(err) - } - return "1 - " + slo_period_burn_rate_ratio + "{" + ruleFilters + "}" -} - -func (s *SLO) RawDashboardInfoQuery() string { - return "vector(1)" -} - -func (s *SLO) RawGoodEventsQuery(w string) (string, error) { - goodConstructedEvents := s.goodEvents.Construct() - var bGood bytes.Buffer - err := simpleQueryTpl.Execute(&bGood, map[string]string{ - "Metric": string(s.goodMetric), - "JobId": string(s.svc), - "Labels": goodConstructedEvents, - "Window": w, - }) - - return bGood.String(), err -} - -func (s *SLO) RawTotalEventsQuery(w string) (string, error) { - totalConstructedEvents := s.totalEvents.Construct() - var bTotal bytes.Buffer - err := simpleQueryTpl.Execute(&bTotal, map[string]string{ - "Metric": string(s.totalMetric), - "JobId": string(s.svc), - "Labels": totalConstructedEvents, - "Window": w, - }) - return bTotal.String(), err -} - -func (s *SLO) RawSLIQuery(w string) (string, error) { - good, err := s.RawGoodEventsQuery(w) - if err != nil { - return "", err - } - total, err := s.RawTotalEventsQuery(w) - if err != nil { - return "", err - } - var bQuery bytes.Buffer - err = rawSliQueryTpl.Execute(&bQuery, SliQueryInfo{ - GoodQuery: good, - TotalQuery: total, - }) - return bQuery.String(), err -} - -func (s *SLO) ConstructRecordingRuleGroup(interval *time.Duration) rulefmt.RuleGroup { - var promInterval prommodel.Duration - var err error - if interval == nil { //sensible default is 1m - promInterval, err = prommodel.ParseDuration(TimeDurationToPromStr(time.Minute)) - - } else { - promInterval, err = prommodel.ParseDuration(TimeDurationToPromStr(*interval)) - if err != nil { - panic(err) - } - } - if err != nil { - panic(err) - } - rrecording := rulefmt.RuleGroup{ - Name: s.GetId() + RecordingRuleSuffix, - Interval: promInterval, - } - for _, w := range NewWindowRange(s.sloPeriod) { - rawSli, err := s.RawSLIQuery(w) - if err != nil { - panic(err) - } - rrecording.Rules = append(rrecording.Rules, rulefmt.RuleNode{ - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_ratio_rate_query_name + w, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: rawSli, - }, - Labels: MergeLabels(s.idLabels, map[string]string{ - slo_window: w, - }, - s.userLabels, - ), - }) - } - return rrecording -} - -func (s *SLO) ConstructMetadataRules(interval *time.Duration) rulefmt.RuleGroup { - var promInterval prommodel.Duration - var err error - if interval == nil { //sensible default is 1m - promInterval, err = prommodel.ParseDuration(TimeDurationToPromStr(time.Minute)) - - } else { - promInterval, err = prommodel.ParseDuration(TimeDurationToPromStr(*interval)) - } - if err != nil { - panic(err) - } - rmetadata := rulefmt.RuleGroup{ - Name: s.GetId() + MetadataRuleSuffix, - Interval: promInterval, - } - rmetadata.Rules = []rulefmt.RuleNode{ - { - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_objective_ratio, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: s.RawObjectiveQuery(), - }, - Labels: MergeLabels( - s.idLabels, - s.userLabels, - ), - }, - { - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_error_budget_ratio, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: s.RawErrorBudgetQuery(), - }, - Labels: MergeLabels( - s.idLabels, - s.userLabels, - ), - }, - { - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_time_period_days, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: s.RawPeriodDurationQuery(), - }, - Labels: MergeLabels( - s.idLabels, - s.userLabels, - ), - }, - { - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_current_burn_rate_ratio, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: s.RawCurrentBurnRateQuery(), - }, - Labels: MergeLabels( - s.idLabels, - s.userLabels, - ), - }, - { - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_period_burn_rate_ratio, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: s.RawPeriodBurnRateQuery(), - }, - Labels: MergeLabels( - s.idLabels, - s.userLabels, - ), - }, - { - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_period_error_budget_remaining_ratio, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: s.RawBudgetRemainingQuery(), - }, - Labels: MergeLabels( - s.idLabels, - s.userLabels, - ), - }, - { - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_info, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: s.RawDashboardInfoQuery(), - }, - Labels: MergeLabels( - s.idLabels, - s.userLabels, - ), - }, - } - return rmetadata -} - -func (s *SLO) AlertPageThreshold() float64 { - return 0.5 -} - -// ConstructAlertingRuleGroup -// -// Note: first two are expected to be the recording rules -// Note: second two are expected to be the alerting rules -func (s *SLO) ConstructAlertingRuleGroup(interval *time.Duration) rulefmt.RuleGroup { - var promInterval prommodel.Duration - var err error - if interval == nil { //sensible default is 1m - promInterval, err = prommodel.ParseDuration(TimeDurationToPromStr(time.Minute)) - - } else { - promInterval, err = prommodel.ParseDuration(TimeDurationToPromStr(*interval)) - } - if err != nil { - panic(err) - } - ralerting := rulefmt.RuleGroup{ - Name: s.GetId() + AlertRuleSuffix, - Interval: promInterval, - } - sloFilters, err := s.GetPrometheusRuleFilterByIdLabels() - if err != nil { - panic(err) - } - sloFilters = "{" + sloFilters + "}" - dur, err := prommodel.ParseDuration(s.sloPeriod) - if err != nil { - panic(err) - } - mwmbWindow := GenerateGoogleWindows(time.Duration(dur)) - var exprTicket bytes.Buffer - errorBudgetRatio := 100 - s.objective - // if errorBudgetRatio == 0 { - //panic(fmt.Sprintf("error budget ratio cannot be treated as 0, from objective : %.9f", s.objective)) - // } - err = mwmbAlertTplBool.Execute(&exprTicket, map[string]string{ - "WindowLabel": slo_window, - "QuickShortMetric": slo_ratio_rate_query_name + "5m", - "QuickShortBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedTicketQuick()), - "QuickLongMetric": slo_ratio_rate_query_name + "30m", - "QuickLongBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedTicketSlow()), - "SlowShortMetric": slo_ratio_rate_query_name + "2h", - "SlowShortBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedTicketQuick()), - "SlowQuickMetric": slo_ratio_rate_query_name + "6h", - "SlowQuickBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedTicketSlow()), - "ErrorBudgetRatio": fmt.Sprintf("%.9f", errorBudgetRatio), - "MetricFilter": sloFilters, - }) - if err != nil { - panic(err) - } - var exprPage bytes.Buffer - err = mwmbAlertTplBool.Execute(&exprPage, map[string]string{ - "WindowLabel": slo_window, - "QuickShortMetric": slo_ratio_rate_query_name + "5m", - "QuickShortBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedPageQuick()), - "QuickLongMetric": slo_ratio_rate_query_name + "30m", - "QuickLongBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedPageSlow()), - "SlowShortMetric": slo_ratio_rate_query_name + "2h", - "SlowShortBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedPageQuick()), - "SlowQuickMetric": slo_ratio_rate_query_name + "6h", - "SlowQuickBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedPageSlow()), - "ErrorBudgetRatio": fmt.Sprintf("%.9f", errorBudgetRatio), - "MetricFilter": sloFilters, - }) - if err != nil { - panic(err) - } - var recordTicket bytes.Buffer - err = mwmbAlertTplBool.Execute(&recordTicket, map[string]string{ - "WindowLabel": slo_window, - "QuickShortMetric": slo_ratio_rate_query_name + "5m", - "QuickShortBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedTicketQuick()), - "QuickLongMetric": slo_ratio_rate_query_name + "30m", - "QuickLongBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedTicketSlow()), - "SlowShortMetric": slo_ratio_rate_query_name + "2h", - "SlowShortBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedTicketQuick()), - "SlowQuickMetric": slo_ratio_rate_query_name + "6h", - "SlowQuickBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedTicketSlow()), - "ErrorBudgetRatio": fmt.Sprintf("%.9f", errorBudgetRatio), - "MetricFilter": sloFilters, - }) - if err != nil { - panic(err) - } - var recordPage bytes.Buffer - err = mwmbAlertTplBool.Execute(&recordPage, map[string]string{ - "WindowLabel": slo_window, - "QuickShortMetric": slo_ratio_rate_query_name + "5m", - "QuickShortBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedPageQuick()), - "QuickLongMetric": slo_ratio_rate_query_name + "30m", - "QuickLongBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedPageSlow()), - "SlowShortMetric": slo_ratio_rate_query_name + "2h", - "SlowShortBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedPageQuick()), - "SlowQuickMetric": slo_ratio_rate_query_name + "6h", - "SlowQuickBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedPageSlow()), - "ErrorBudgetRatio": fmt.Sprintf("%.9f", errorBudgetRatio), - "MetricFilter": sloFilters, - }) - if err != nil { - panic(err) - } - - // Note: first two are expected to be the recording rules - ralerting.Rules = append(ralerting.Rules, rulefmt.RuleNode{ - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_alert_ticket_window, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: recordTicket.String(), - }, - Labels: MergeLabels(s.idLabels, map[string]string{"slo_severity": "ticket"}, s.userLabels), - }) - ralerting.Rules = append(ralerting.Rules, rulefmt.RuleNode{ - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_alert_page_window, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: recordPage.String(), - }, - Labels: MergeLabels(s.idLabels, map[string]string{"slo_severity": "page"}, s.userLabels), - }) - - // note: second two are expected to be the alerting rules - arPage := metrics.AlertingRule{ - Expr: exprPage.String(), - Labels: MergeLabels(s.idLabels, map[string]string{"slo_severity": "page"}, s.userLabels), - } - arTicket := metrics.AlertingRule{ - Expr: exprTicket.String(), - Labels: MergeLabels(s.idLabels, map[string]string{"slo_severity": "ticket"}, s.userLabels), - } - arPageRule, err := arPage.Build(metrics.WithSloId(s.GetId(), "page", AlertRuleSuffix)) - if err != nil { - panic(err) - } - - arTicketRule, err := arTicket.Build(metrics.WithSloId(s.GetId(), "ticket", AlertRuleSuffix)) - if err != nil { - panic(err) - } - ralerting.Rules = append(ralerting.Rules, *arPageRule) - ralerting.Rules = append(ralerting.Rules, *arTicketRule) - return ralerting -} - -func (s *SLO) ConstructCortexRules(interval *time.Duration) (sli, metadata, alerts rulefmt.RuleGroup) { - rrecording := s.ConstructRecordingRuleGroup(interval) - rmetadata := s.ConstructMetadataRules(interval) - ralerts := s.ConstructAlertingRuleGroup(interval) - - _, debug := os.LookupEnv("DEBUG_RULES") - if debug { - srecording, err := yaml.Marshal(rrecording) - if err != nil { - panic(err) - } - smetadata, err := yaml.Marshal(rmetadata) - if err != nil { - panic(err) - } - salerts, err := yaml.Marshal(ralerts) - if err != nil { - panic(err) - } - os.WriteFile(s.GetId()+"-recording.yaml", srecording, 0644) - os.WriteFile(s.GetId()+"-metadata.yaml", smetadata, 0644) - os.WriteFile(s.GetId()+"-alerts.yaml", salerts, 0644) - } - return rrecording, rmetadata, ralerts -} - -func (s *SLO) ConstructRawAlertQueries() (yaml.Node, yaml.Node) { - filters, err := s.GetPrometheusRuleFilterByIdLabels() - if err != nil { - panic(err) - } - filters = "{" + filters + "}" - alertCriticalRawQuery := s.ConstructAlertingRuleGroup(lo.ToPtr(sloapi.MinEvaluateInterval)).Rules[0].Expr - alertCriticalRawQuery.Value = strings.Replace(alertCriticalRawQuery.Value, filters, "", -1) - alertCriticalRawQuery.Value = strings.Replace(alertCriticalRawQuery.Value, fmt.Sprintf("without (%s)", slo_window), "", -1) - alertSevereRawQuery := s.ConstructAlertingRuleGroup(lo.ToPtr(sloapi.MinEvaluateInterval)).Rules[1].Expr - alertSevereRawQuery.Value = strings.Replace(alertSevereRawQuery.Value, filters, "", -1) - alertSevereRawQuery.Value = strings.Replace(alertSevereRawQuery.Value, fmt.Sprintf("without (%s)", slo_window), "", -1) - - for _, rule := range s.ConstructRecordingRuleGroup(nil).Rules { - alertCriticalRawQuery.Value = strings.Replace(alertCriticalRawQuery.Value, rule.Record.Value, rule.Expr.Value, -1) - alertSevereRawQuery.Value = strings.Replace(alertSevereRawQuery.Value, rule.Record.Value, rule.Expr.Value, -1) - } - - _, err = parser.ParseExpr(alertCriticalRawQuery.Value) - if err != nil { - panic(err) - } - _, err = parser.ParseExpr(alertSevereRawQuery.Value) - if err != nil { - panic(err) - } - return alertCriticalRawQuery, alertSevereRawQuery -} - -func NewWindowRange(sloPeriod string) []string { - return []string{"5m", "30m", "1h", "2h", "6h", "1d", sloPeriod} -} - -// DetectActiveWindows -// -// @warning Expectation is that the timestamps are ordered when traversing -// matrix --> sample streams --> [] values -// but this may not always be the case -func DetectActiveWindows(severity string, matrix *prommodel.Matrix) ([]*sloapi.AlertFiringWindows, error) { - returnWindows := []*sloapi.AlertFiringWindows{} - if matrix == nil { - return nil, fmt.Errorf("Got empty alerting window %s matrix from cortex", severity) - } - for _, row := range *matrix { - for _, rowValue := range row.Values { - ts := time.Unix(rowValue.Timestamp.Unix(), 0) - // rising edge - if rowValue.Value == 1 { - if len(returnWindows) == 0 { - returnWindows = append(returnWindows, &sloapi.AlertFiringWindows{ - Start: timestamppb.New(ts), - End: nil, - Severity: severity, - }) - } else { - if returnWindows[len(returnWindows)-1].End != nil { - returnWindows = append(returnWindows, &sloapi.AlertFiringWindows{ - Start: timestamppb.New(ts), - End: nil, - Severity: severity, - }) - } - } - } else { //falling edge - if len(returnWindows) > 0 && returnWindows[len(returnWindows)-1].End == nil { - returnWindows[len(returnWindows)-1].End = timestamppb.New(ts) - } - } - } - } - if len(returnWindows) > 0 { - if returnWindows[len(returnWindows)-1].End == nil { - returnWindows[len(returnWindows)-1].End = timestamppb.New(time.Now()) - } - } - return returnWindows, nil -} - -// ToMatchingSubsetIdenticalMetric only applies when the good metric & total metric id is the same -func ToMatchingSubsetIdenticalMetric(goodEvents, totalEvents []*sloapi.Event) (good, total []*sloapi.Event) { - indexGood := map[string][]string{} - for _, g := range goodEvents { - indexGood[g.Key] = g.Vals - } - for _, t := range totalEvents { - if t.Key == "" || t.Vals == nil { - continue - } - if _, ok := indexGood[t.Key]; ok { // event type defined on good and total so reconcile - t.Vals = LeftJoinSlice(indexGood[t.Key], t.Vals) - } else { - // event type not defined on good, so coerce good to have it, otherwise - goodEvents = append(goodEvents, t) - } - } - return goodEvents, totalEvents -} - -func MergeRuleGroups(left rulefmt.RuleGroup, right *rulefmt.RuleGroup) *rulefmt.RuleGroup { - newRules := LeftJoinSliceAbstract[rulefmt.RuleNode, string]( - left.Rules, - right.Rules, - func(r rulefmt.RuleNode) string { return r.Record.Value }, - ) - return &rulefmt.RuleGroup{ - Name: left.Name, - Interval: left.Interval, - Rules: newRules, - } -} diff --git a/plugins/slo/pkg/slo/slo_suite_test.go b/plugins/slo/pkg/slo/slo_suite_test.go deleted file mode 100644 index 282a64f7e5..0000000000 --- a/plugins/slo/pkg/slo/slo_suite_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package slo_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - _ "github.com/rancher/opni/pkg/test/setup" - _ "github.com/rancher/opni/plugins/alerting/test" - _ "github.com/rancher/opni/plugins/metrics/test" -) - -func TestSlo(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Slo Suite") -} diff --git a/plugins/slo/pkg/slo/slo_test.go b/plugins/slo/pkg/slo/slo_test.go deleted file mode 100644 index d7bbbbec8e..0000000000 --- a/plugins/slo/pkg/slo/slo_test.go +++ /dev/null @@ -1,826 +0,0 @@ -package slo_test - -import ( - "context" - "fmt" - "math/rand" - "strings" - "time" - - "github.com/goombaio/namegenerator" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - prommodel "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/rulefmt" - "github.com/prometheus/prometheus/promql/parser" - capabilityv1 "github.com/rancher/opni/pkg/apis/capability/v1" - corev1 "github.com/rancher/opni/pkg/apis/core/v1" - managementv1 "github.com/rancher/opni/pkg/apis/management/v1" - storagev1 "github.com/rancher/opni/pkg/apis/storage/v1" - - "github.com/rancher/opni/pkg/capabilities/wellknown" - "github.com/rancher/opni/pkg/metrics/compat" - "github.com/rancher/opni/pkg/test" - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - "github.com/rancher/opni/plugins/metrics/apis/cortexops" - sloapi "github.com/rancher/opni/plugins/slo/apis/slo" - "github.com/rancher/opni/plugins/slo/pkg/slo" - "google.golang.org/protobuf/types/known/durationpb" - "google.golang.org/protobuf/types/known/emptypb" - "gopkg.in/yaml.v3" -) - -// need to check always that good <= total -func expectValidEventSubsets(good []*sloapi.Event, total []*sloapi.Event) { - cacheTotal := make(map[string]bool) - listVals := make(map[string][]string) - for i := 0; i < len(good); i++ { - listVals[good[i].Key] = good[i].Vals - } - for i := 0; i < len(total); i++ { - cacheTotal[total[i].Key] = false - Expect(len(total[i].Vals)).To(BeNumerically(">=", len(listVals[total[i].Key]))) - Expect(total[i].Vals).To(ContainElements(listVals[total[i].Key])) // total events must be at least a super set of good events - } - for i := 0; i < len(good); i++ { - if _, ok := cacheTotal[good[i].Key]; ok { - cacheTotal[good[i].Key] = true - } - } - - // everything in total should be defined in good - for _, v := range cacheTotal { - Expect(v).To(BeTrue()) - } - -} - -var _ = Describe("Converting SLO information to Cortex rules", Ordered, Label("integration", "slow"), func() { - sloObj := slo.NewSLO( - "slo-name", - "30d", - 99.99, - "prometheus", - "prometheus_http_requests_total", - "prometheus_http_requests_total", - map[string]string{ - "important": "true", - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200"}, - }, - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200", "500", "503"}, - }, - }, - ) - // test environment references - var env *test.Environment - var pPort int - var adminClient cortexadmin.CortexAdminClient - - BeforeAll(func() { - env = &test.Environment{} - Expect(env.Start()).To(Succeed()) - DeferCleanup(env.Stop) - - client := env.NewManagementClient() - token, err := client.CreateBootstrapToken(env.Context(), &managementv1.CreateBootstrapTokenRequest{ - Ttl: durationpb.New(time.Hour), - }) - Expect(err).NotTo(HaveOccurred()) - info, err := client.CertsInfo(env.Context(), &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - opsClient := cortexops.NewCortexOpsClient(env.ManagementClientConn()) - _, err = opsClient.ConfigureCluster(context.Background(), &cortexops.ClusterConfiguration{ - Mode: cortexops.DeploymentMode_AllInOne, - Storage: &storagev1.StorageSpec{ - Backend: storagev1.Filesystem, - }, - }) - Expect(err).NotTo(HaveOccurred()) - - _, errC := env.StartAgent("agent", token, []string{info.Chain[len(info.Chain)-1].Fingerprint}, test.WithContext(env.Context())) - Eventually(errC).Should(Receive(BeNil())) - _, errC = env.StartAgent("agent2", token, []string{info.Chain[len(info.Chain)-1].Fingerprint}, test.WithContext(env.Context())) - Eventually(errC).Should(Receive(BeNil())) - - _, err = client.InstallCapability(env.Context(), &managementv1.CapabilityInstallRequest{ - Name: wellknown.CapabilityMetrics, - Target: &capabilityv1.InstallRequest{ - Cluster: &corev1.Reference{Id: "agent"}, - }, - }) - - Expect(err).NotTo(HaveOccurred()) - _, err = client.InstallCapability(env.Context(), &managementv1.CapabilityInstallRequest{ - Name: wellknown.CapabilityMetrics, - Target: &capabilityv1.InstallRequest{ - Cluster: &corev1.Reference{Id: "agent2"}, - }, - }) - Expect(err).NotTo(HaveOccurred()) - - adminClient = cortexadmin.NewCortexAdminClient(env.ManagementClientConn()) - Eventually(func() error { - stats, err := adminClient.AllUserStats(context.Background(), &emptypb.Empty{}) - if err != nil { - return err - } - for _, item := range stats.Items { - if item.UserID == "agent" { - if item.NumSeries > 0 { - return nil - } - } - } - return fmt.Errorf("waiting for metric data to be stored in cortex") - }, 30*time.Second, 1*time.Second).Should(Succeed()) - Eventually(func() error { - stats, err := adminClient.AllUserStats(context.Background(), &emptypb.Empty{}) - if err != nil { - return err - } - for _, item := range stats.Items { - if item.UserID == "agent2" { - if item.NumSeries > 0 { - return nil - } - } - } - return fmt.Errorf("waiting for metric data to be stored in cortex") - }, 30*time.Second, 1*time.Second).Should(Succeed()) - }) - - When("We receive alert matrix data from cortex", func() { - It("should be able to convert the alert matrix data to active windows", func() { - _, err := slo.DetectActiveWindows("severe", nil) - Expect(err).To(HaveOccurred()) - - start := time.Now().Add(-(time.Hour * 24)) - startAlertWindow1 := time.Now().Add(-time.Hour * 5) - endAlertWindow1 := time.Now().Add(-time.Hour * 4) - startAlertWindow2 := time.Now().Add(-time.Hour * 3) - endAlertWindow2 := time.Now().Add(-time.Hour * 2) - - matrix := &prommodel.Matrix{ - // sample stream 1 - { - Values: []prommodel.SamplePair{ - { - Timestamp: prommodel.TimeFromUnix(start.Unix()), - Value: 0, - }, - { - Timestamp: prommodel.TimeFromUnix(startAlertWindow1.Unix()), - Value: 1, - }, - }, - }, - // sample stream 2 - { - Values: []prommodel.SamplePair{ - { - Timestamp: prommodel.TimeFromUnix(endAlertWindow1.Unix()), - Value: 0, - }, - { - Timestamp: prommodel.TimeFromUnix(endAlertWindow1.Add(time.Minute).Unix()), - Value: 0, - }, - { - Timestamp: prommodel.TimeFromUnix(startAlertWindow2.Unix()), - Value: 1, - }, - { - Timestamp: prommodel.TimeFromUnix(startAlertWindow2.Add(time.Minute).Unix()), - Value: 1, - }, - { - Timestamp: prommodel.TimeFromUnix(endAlertWindow2.Unix()), - Value: 0, - }, - }, - }, - } - windows, err := slo.DetectActiveWindows("severe", matrix) - Expect(err).NotTo(HaveOccurred()) - Expect(windows).To(HaveLen(2)) - Expect(windows[0].Start.AsTime().Unix()).To(Equal(startAlertWindow1.Unix())) - Expect(windows[0].End.AsTime().Unix()).To(Equal(endAlertWindow1.Unix())) - Expect(windows[1].Start.AsTime().Unix()).To(Equal(startAlertWindow2.Unix())) - Expect(windows[1].End.AsTime().Unix()).To(Equal(endAlertWindow2.Unix())) - }) - }) - - When("We receive SLO event data from the user", func() { - It("Should be able to convert the SLO event data to matching subsets for identical metric names", func() { - // total events empty ==> total events stays empty - goodEvents1 := []*sloapi.Event{ - { - Key: "code", - Vals: []string{"200"}, - }, - } - totalEvents1 := []*sloapi.Event{} - g, t := slo.ToMatchingSubsetIdenticalMetric(goodEvents1, totalEvents1) - Expect(g).To(Equal(goodEvents1)) - Expect(t).To(Equal(totalEvents1)) - - // management api server will pass in empty events to an empty list, - // because it hates us - totalEvents1 = []*sloapi.Event{{}} // in this case, this is what it looks like - g, t = slo.ToMatchingSubsetIdenticalMetric(goodEvents1, totalEvents1) - Expect(g).To(Equal(goodEvents1)) - Expect(t).To(Equal(totalEvents1)) - - // to be dilligent we should also check when t.Vals is nil - totalEvents1 = []*sloapi.Event{{Key: "code"}} - g, t = slo.ToMatchingSubsetIdenticalMetric(goodEvents1, totalEvents1) - Expect(g).To(Equal(goodEvents1)) - Expect(t).To(Equal(totalEvents1)) - - // should fill in missing subset - goodEvents2 := []*sloapi.Event{ - { - Key: "code", - Vals: []string{"200"}, - }, - } - totalEvents2 := []*sloapi.Event{ - { - Key: "code", - Vals: []string{"500", "503"}, - }, - } - goodEvents2Copy := make([]string, len(goodEvents2[0].Vals)) - copy(goodEvents2Copy, goodEvents2[0].Vals) - totalEvents2Copy := make([]string, len(totalEvents2[0].Vals)) - copy(totalEvents2Copy, totalEvents2[0].Vals) - g, t = slo.ToMatchingSubsetIdenticalMetric(goodEvents2, totalEvents2) - Expect(g).To(Equal(goodEvents2)) - Expect(t[0].Key).To(Equal(totalEvents2[0].Key)) - Expect(t[0].Vals).To(ConsistOf(append(goodEvents2Copy, totalEvents2Copy...))) - - // should coerce subsets with different filters - goodEvents3 := []*sloapi.Event{ - { - Key: "code", - Vals: []string{"200"}, - }, - } - totalEvents3 := []*sloapi.Event{ - { - Key: "handler", - Vals: []string{"/ready"}, - }, - } - g, t = slo.ToMatchingSubsetIdenticalMetric(goodEvents3, totalEvents3) - Expect(g).To(Equal([]*sloapi.Event{ - { - Key: "code", - Vals: []string{"200"}, - }, - { - Key: "handler", - Vals: []string{"/ready"}, - }, - })) - Expect(t).To(Equal(totalEvents3)) - - goodEvents4 := []*sloapi.Event{{ - Key: "code", - Vals: []string{"200"}}} - totalEvents4 := []*sloapi.Event{ - { - Key: "code", - Vals: []string{"500", "503"}, - }, - } - - g, t = slo.ToMatchingSubsetIdenticalMetric(goodEvents4, totalEvents4) - Expect(Expect(g[0].Key).To(Equal(goodEvents4[0].Key))) - Expect(g[0].Vals).To(ConsistOf([]string{"200"})) - Expect(t[0].Vals).To(ConsistOf(slo.LeftJoinSlice(goodEvents4[0].Vals, totalEvents4[0].Vals))) - - }) - - Specify("The event matching subset algorithm should be robust to chaos testing", func() { - // FIXME: this test is kinda hacky and poorly written - for numTests := 0; numTests < 15; numTests++ { - goodEventNum := rand.Intn(10) + 1 - goodEventValNum := rand.Intn(1000) + 1 - totalEventNum := rand.Intn(10) + 1 - - goodEvents := make([]*sloapi.Event, goodEventNum) - totalEvents := make([]*sloapi.Event, totalEventNum) - goodEventCache := make(map[string]struct{}) - goodEventValsCache := make(map[string]map[string]struct{}) - gen := namegenerator.NewNameGenerator(time.Now().UnixNano()) - for i := 0; i < goodEventNum; i++ { - name := gen.Generate() - vals := []string{} - for j := 0; j < goodEventValNum; j++ { - newVal := gen.Generate() - vals = append(vals, newVal) - if _, ok := goodEventValsCache[name]; !ok { - goodEventValsCache[name] = make(map[string]struct{}) - } - goodEventValsCache[name][newVal] = struct{}{} - } - goodEvents[i] = &sloapi.Event{ - Key: name, - Vals: vals, - } - goodEventCache[name] = struct{}{} - } - for i := 0; i < totalEventNum; i++ { - isMany := rand.Intn(2) - var totalEventValNum int - if isMany == 0 { - totalEventValNum = rand.Intn(1000) - } else { - totalEventValNum = rand.Intn(3) - } - - // 50% chance to use an existing key from goodEvents - var code string - var vals []string - if rand.Intn(2) == 0 && len(goodEventCache) > 0 { // use existing key - if goodEventNum < 0 { - panic(goodEvents) - } - index := rand.Intn(goodEventNum) - code = goodEvents[index].Key - delete(goodEventCache, code) - for j := 0; j < totalEventValNum; j++ { - //25% chance to use existing value from goodEvents - if rand.Intn(4) == 0 && len(goodEventValsCache[code]) > 0 { - keys := make([]string, len(goodEventValsCache[code])) - - x := 0 - for k := range goodEventValsCache[code] { - keys[x] = k - x++ - } - index := rand.Intn(len(keys)) - vals = append(vals, keys[index]) - delete(goodEventValsCache[code], keys[index]) - } else { - vals = append(vals, gen.Generate()) - } - } - } else { //create new key - name := gen.Generate() - code = name - for j := 0; j < totalEventValNum; j++ { - vals = append(vals, gen.Generate()) - } - } - totalEvents[i] = &sloapi.Event{ - Key: code, - Vals: vals, - } - } - invalid := false - for i := 0; i < len(totalEvents); i++ { - if totalEvents[i].Vals == nil || len(totalEvents[i].Vals) == 0 { - invalid = true - break - } - if totalEvents[i] == nil { - invalid = true - break - } - } - if invalid == true { - continue // FIXME: not sure why we are getting invalid constructions - } - newGoodEvents, newTotalEvents := slo.ToMatchingSubsetIdenticalMetric(goodEvents, totalEvents) - expectValidEventSubsets(newGoodEvents, newTotalEvents) - } - }) - }) - - When("We use SLO objects to construct rules", func() { - Specify("Label pairs should be able to construct promQL filters", func() { - fmt.Println(pPort) - goodLabelPairs := slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200"}, - }, - } - Expect(goodLabelPairs.Construct()).To(Equal(",code=~\"200\"")) - totalLabelPairs := slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200", "500", "503"}, - }, - } - Expect(totalLabelPairs.Construct()).To(Equal(",code=~\"200|500|503\"")) - }) - - It("Should construct an SLO object", func() { - sloObj := slo.NewSLO( - "slo-name", - "30d", - 99.99, - "prometheus", - slo.Metric("prometheus_http_requests_total"), - slo.Metric("prometheus_http_requests_total"), - map[string]string{ - "important": "true", - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200"}, - }, - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200", "500", "503"}, - }, - }, - ) - Expect(sloObj).ToNot(BeNil()) - Expect(sloObj.GetId()).ToNot(BeNil()) - sloObj2 := slo.SLOFromId( - - "slo-name", - "30d", - 99.99, - "prometheus", - "", - "slo-operator", - map[string]string{ - "important": "true", - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200"}, - }, - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200", "500", "503"}, - }, - }, - sloObj.GetId(), - ) - Expect(sloObj2).ToNot(BeNil()) - Expect(sloObj2.GetId()).To(Equal(sloObj.GetId())) - }) - Specify("SLO objects should be able to create valid SLI Prometheus rules", func() { - sloObj := slo.NewSLO( - "slo-name", - "30d", - 99.99, - "prometheus", - "prometheus_http_requests_total", - "prometheus_http_requests_total", - map[string]string{ - "important": "true", - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200"}, - }, - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200", "500", "503"}, - }, - }, - ) - query, err := sloObj.RawSLIQuery("5m") - Expect(err).To(Succeed()) - Expect(query).NotTo(Equal("")) - _, err = parser.ParseExpr(query) - Expect(err).To(Succeed()) - }) - - Specify("SLO objects should be able to create valid metadata Prometheus rules", func() { - sloObj := slo.NewSLO( - "slo-name", - "30d", - 99.99, - "prometheus", - "prometheus_http_requests_total", - "prometheus_http_requests_total", - map[string]string{ - "important": "true", - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200"}, - }, - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200", "500", "503"}, - }, - }, - ) - dash := sloObj.RawDashboardInfoQuery() - _, err := parser.ParseExpr(dash) - Expect(err).To(Succeed()) - rawBudget := sloObj.RawErrorBudgetQuery() - _, err = parser.ParseExpr(rawBudget) - Expect(err).To(Succeed()) - - rawObjective := sloObj.RawObjectiveQuery() - _, err = parser.ParseExpr(rawObjective) - Expect(err).To(Succeed()) - rawRemainingBudget := sloObj.RawBudgetRemainingQuery() - _, err = parser.ParseExpr(rawRemainingBudget) - Expect(err).To(Succeed()) - rawPeriodAsVectorDays := sloObj.RawPeriodDurationQuery() - _, err = parser.ParseExpr(rawPeriodAsVectorDays) - Expect(err).To(Succeed()) - // burn rate - curBurnRate := sloObj.RawCurrentBurnRateQuery() - _, err = parser.ParseExpr(curBurnRate) - Expect(err).To(Succeed()) - periodBurnRate := sloObj.RawPeriodBurnRateQuery() - _, err = parser.ParseExpr(periodBurnRate) - Expect(err).To(Succeed()) - }) - - Specify("SLO objects should be able to create valid alerting Prometheus rules", func() { - interval := time.Second - ralerts := sloObj.ConstructAlertingRuleGroup(&interval) - for _, alertRule := range ralerts.Rules { - _, err := parser.ParseExpr(alertRule.Expr.Value) - Expect(err).To(Succeed()) - } - }) - - Specify("SLO objects should be able to create valid Cortex recording rule groups", func() { - interval := time.Second - rrecording := sloObj.ConstructRecordingRuleGroup(&interval) - out, err := yaml.Marshal(rrecording) - Expect(err).To(Succeed()) - y := rulefmt.RuleGroup{} - err = yaml.Unmarshal(out, &y) - Expect(err).To(Succeed()) - //FIXME: when joe is back, should add pkg/rules to his fork of cortex-tools - //errors := rules.ValidateRuleGroup(y) - //Expect(errors).To(BeEmpty()) - - }) - - Specify("SLO objects should be able to create valid Cortex metadata rule groups", func() { - interval := time.Second - rmetadata := sloObj.ConstructMetadataRules(&interval) - out, err := yaml.Marshal(rmetadata) - Expect(err).To(Succeed()) - y := rulefmt.RuleGroup{} - err = yaml.Unmarshal(out, &y) - Expect(err).To(Succeed()) - //FIXME: when joe is back, should add pkg/rules to his fork of cortex-tools - //errors := rules.ValidateRuleGroup(y) - //Expect(errors).To(BeEmpty()) - - }) - - Specify("SLO objects should be able to create valid Cortex alerting rule groups", func() { - interval := time.Second - ralerts := sloObj.ConstructMetadataRules(&interval) - out, err := yaml.Marshal(ralerts) - Expect(err).To(Succeed()) - y := rulefmt.RuleGroup{} - err = yaml.Unmarshal(out, &y) - Expect(err).To(Succeed()) - //FIXME: when joe is back, should add pkg/rules to his fork of cortex-tools - //errors := rules.ValidateRuleGroup(y) - //Expect(errors).To(BeEmpty()) - }) - }) - - When("When use raw SLO constructions with cortex admin client", func() { - Specify("The individual parts of the raw SLI queries should return data from cortex", func() { - // needed to ensure that prometheus agent registers a 200 status code - // otherwise the test is flaky - rawGood, err := sloObj.RawGoodEventsQuery("5m") - Expect(err).NotTo(HaveOccurred()) - rawTotal, err := sloObj.RawTotalEventsQuery("5m") - Expect(err).NotTo(HaveOccurred()) - Eventually(func() error { - return nil - }, time.Second*90, time.Second*30).Should(BeNil()) - respGood, err := adminClient.Query(env.Context(), &cortexadmin.QueryRequest{ - Query: rawGood, - Tenants: []string{"agent"}, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(respGood.Data).NotTo(BeEmpty()) - qres, err := compat.UnmarshalPrometheusResponse(respGood.Data) - Expect(err).NotTo(HaveOccurred()) - goodVector, err := qres.GetVector() - Expect(err).NotTo(HaveOccurred()) - Expect(goodVector).NotTo(BeNil()) - Expect(*goodVector).NotTo(BeEmpty()) - for _, sample := range *goodVector { - Expect(sample.Value).To(BeNumerically(">=", 0)) - Expect(sample.Timestamp).To(BeNumerically(">=", 0)) - } - - respTotal, err := adminClient.Query(env.Context(), &cortexadmin.QueryRequest{ - Query: rawTotal, - Tenants: []string{"agent"}, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(respTotal.Data).NotTo(BeEmpty()) - qresTotal, err := compat.UnmarshalPrometheusResponse(respTotal.Data) - Expect(err).NotTo(HaveOccurred()) - totalVector, err := qresTotal.GetVector() - Expect(err).NotTo(HaveOccurred()) - Expect(totalVector).NotTo(BeNil()) - Expect(*totalVector).NotTo(BeEmpty()) - for _, sample := range *totalVector { - Expect(sample.Value).To(BeNumerically(">=", 0)) - Expect(sample.Timestamp).To(BeNumerically(">=", 0)) - } - - adHocSliErrorRatioQuery := fmt.Sprintf(" 1 - (%s)/(%s)", rawGood, rawTotal) - respSli, err := adminClient.Query(env.Context(), &cortexadmin.QueryRequest{ - Tenants: []string{"agent"}, - Query: adHocSliErrorRatioQuery, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(respSli.Data).NotTo(BeEmpty()) - qresSli, err := compat.UnmarshalPrometheusResponse(respSli.Data) - Expect(err).NotTo(HaveOccurred()) - sliErrorRatioVector, err := qresSli.GetVector() - Expect(err).NotTo(HaveOccurred()) - Expect(sliErrorRatioVector).NotTo(BeNil()) - Expect(*sliErrorRatioVector).NotTo(BeEmpty()) - for _, sample := range *sliErrorRatioVector { - Expect(sample.Value).To(BeNumerically(">=", 0)) - Expect(sample.Value).To(BeNumerically("<=", 1)) - Expect(sample.Timestamp).To(BeNumerically(">=", 0)) - } - }) - - Specify("All of the raw SLI queries should return data from cortex", func() { - time.Sleep(time.Second * 10) - interval := time.Second - rrecording := sloObj.ConstructRecordingRuleGroup(&interval) - for _, rawRule := range rrecording.Rules { - resp, err := adminClient.Query(env.Context(), &cortexadmin.QueryRequest{ - Query: rawRule.Expr.Value, - Tenants: []string{"agent"}, - }) - Expect(err).NotTo(HaveOccurred()) - rawBytes := resp.Data - qres, err := compat.UnmarshalPrometheusResponse(rawBytes) - Expect(err).NotTo(HaveOccurred()) - Expect(qres).NotTo(BeNil()) - recordingVector, err := qres.GetVector() - Expect(err).NotTo(HaveOccurred()) - Expect(recordingVector).NotTo(BeNil()) - Expect(*recordingVector).NotTo(BeEmpty()) - for _, sample := range *recordingVector { - Expect(sample.Value).To(BeNumerically(">=", 0)) - Expect(sample.Timestamp).To(BeNumerically(">=", 0)) - } - } - }) - - Specify("All of the raw Metadata queries should return data from cortex", func() { - // TODO : need to replace all rule names in each of these Expr's to their real expr - }) - - Specify("All of the raw alert queries should return data from cortex", func() { - // TODO: need to replace all rule names in each of these Expr's to their real expr - }) - - Specify("After aplying the rules to cortex, each rule name should evaluate to non-empty data", func() { - // Manually apply the SLI recording rules - interval := time.Second - rrecording := sloObj.ConstructRecordingRuleGroup(&interval) - rmetadata := sloObj.ConstructMetadataRules(&interval) - ralerts := sloObj.ConstructAlertingRuleGroup(&interval) - - outRecording, err := yaml.Marshal(rrecording) - Expect(err).To(Succeed()) - - _, err = adminClient.LoadRules(env.Context(), &cortexadmin.LoadRuleRequest{ - ClusterId: "agent", - Namespace: "slo", - YamlContent: outRecording, - }) - Expect(err).NotTo(HaveOccurred()) - outMetadata, err := yaml.Marshal(rmetadata) - Expect(err).NotTo(HaveOccurred()) - _, err = adminClient.LoadRules(env.Context(), &cortexadmin.LoadRuleRequest{ - ClusterId: "agent", - Namespace: "slo", - YamlContent: outMetadata, - }) - Expect(err).NotTo(HaveOccurred()) - outAlerts, err := yaml.Marshal(ralerts) - Expect(err).NotTo(HaveOccurred()) - _, err = adminClient.LoadRules(env.Context(), &cortexadmin.LoadRuleRequest{ - ClusterId: "agent", - Namespace: "slo", - YamlContent: outAlerts, - }) - Expect(err).NotTo(HaveOccurred()) - - Eventually(func() error { - resp, err := adminClient.ListRules(env.Context(), &cortexadmin.ListRulesRequest{ - ClusterId: []string{"agent"}, - }) - if err != nil { - return err - } - for _, group := range resp.Data.Groups { - if strings.Contains(group.Name, sloObj.GetId()) { - return nil - } - } - return nil - }, time.Minute*2, time.Second*30).Should(Succeed()) - - // check the recording rule names to make sure they return data - Eventually(func() error { - for _, rawRule := range rrecording.Rules { - ruleVector, err := slo.QuerySLOComponentByRawQuery(env.Context(), adminClient, rawRule.Expr.Value, "agent") - if err != nil { - return err - } - - if ruleVector == nil || len(*ruleVector) == 0 { - return fmt.Errorf("expect rule vector to contain data") - } - for _, sample := range *ruleVector { - if sample.Timestamp == 0 { - return fmt.Errorf("expect sample.Timestamp to contain data") - } - } - } - return nil - }, time.Minute*2, time.Second*30).Should(Succeed()) - - Eventually(func() error { - for _, rawRule := range rmetadata.Rules { - ruleVector, err := slo.QuerySLOComponentByRecordName(env.Context(), adminClient, rawRule.Record.Value, "agent") - if err != nil { - return err - } - - if ruleVector == nil || len(*ruleVector) == 0 { - return fmt.Errorf("expect rule vector to contain data") - } - for _, sample := range *ruleVector { - if sample.Timestamp == 0 { - return fmt.Errorf("expect sample.Timestamp to contain data") - } - } - } - return nil - }, time.Minute*2, time.Second*30).Should(Succeed()) - - Eventually(func() error { - rawSevereAlertQuery, rawCriticalAlertQuery := sloObj.ConstructRawAlertQueries() - // @debug - //rawSevereAlertQueryComponents := strings.Split(rawSevereAlertQuery, "or") - //rawCriticalAlertQueryComponents := strings.Split(rawCriticalAlertQuery, "or") - //var alertRules []string - //alertRules = append(alertRules, rawSevereAlertQueryComponents...) - //alertRules = append(alertRules, rawCriticalAlertQueryComponents...) - alertRules := []string{rawSevereAlertQuery.Value, rawCriticalAlertQuery.Value} - for _, rawRule := range alertRules { - ruleVector, err := slo.QuerySLOComponentByRawQuery(env.Context(), adminClient, rawRule, "agent") - if err != nil { - return err - } - if ruleVector == nil || len(*ruleVector) == 0 { - return fmt.Errorf("expect rule vector to contain data") - } - for _, sample := range *ruleVector { - if sample.Timestamp == 0 { - return fmt.Errorf("expect sample.Timestamp to contain data") - } - } - } - return nil - }, time.Minute*2, time.Second*30).Should(Succeed()) - }) - }) -}) diff --git a/plugins/slo/pkg/slo/system.go b/plugins/slo/pkg/slo/system.go deleted file mode 100644 index 8b8a74a0df..0000000000 --- a/plugins/slo/pkg/slo/system.go +++ /dev/null @@ -1,48 +0,0 @@ -package slo - -import ( - "os" - - alertingv1 "github.com/rancher/opni/pkg/apis/alerting/v1" - managementv1 "github.com/rancher/opni/pkg/apis/management/v1" - "github.com/rancher/opni/pkg/plugins/apis/system" - "github.com/rancher/opni/pkg/slo/shared" - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - sloapi "github.com/rancher/opni/plugins/slo/apis/slo" -) - -func (p *Plugin) UseManagementAPI(client managementv1.ManagementClient) { - p.mgmtClient.Set(client) - <-p.ctx.Done() -} - -func (p *Plugin) UseKeyValueStore(client system.KeyValueStoreClient) { - p.storage.Set(StorageAPIs{ - SLOs: system.NewKVStoreClient[*sloapi.SLOData](client), - Services: system.NewKVStoreClient[*sloapi.Service](client), - Metrics: system.NewKVStoreClient[*sloapi.Metric](client), - }) - <-p.ctx.Done() -} - -func (p *Plugin) UseAPIExtensions(intf system.ExtensionClientInterface) { - cc, err := intf.GetClientConn(p.ctx, "CortexAdmin", "AlertEndpoints") - if err != nil { - p.logger.Error("failed to get cortex admin client", "error", err) - if p.ctx.Err() != nil { - // Plugin is shutting down, don't exit - return - } - os.Exit(1) - } - adminClient := cortexadmin.NewCortexAdminClient(cc) - alertingEndpointClient := alertingv1.NewAlertEndpointsClient(cc) - - p.adminClient.Set(adminClient) - p.alertEndpointClient.Set(alertingEndpointClient) - RegisterDatasource( - shared.MonitoringDatasource, - NewSLOMonitoringStore(p, p.logger), - NewMonitoringServiceBackend(p, p.logger), - ) -} diff --git a/plugins/slo/pkg/slo/util.go b/plugins/slo/pkg/slo/util.go deleted file mode 100644 index 31c7545cbb..0000000000 --- a/plugins/slo/pkg/slo/util.go +++ /dev/null @@ -1,68 +0,0 @@ -package slo - -import ( - "text/template" - "time" - - prommodel "github.com/prometheus/common/model" -) - -const ( - tplKeyWindow = "window" -) - -// Reference : https://github.com/slok/sloth/blob/eddd8145a696c3dc6d423e9d50cdb906186a52a3/internal/prometheus/recording_rules.go#L308 -var burnRateRecordingExprTpl = template.Must(template.New("burnRateExpr").Option("missingkey=error").Parse(`{{ .SLIErrorMetric }}{{ .MetricFilter }} -/ on({{ .SLOIDName }}, {{ .SLOLabelName }}, {{ .SLOServiceName }}) group_left -{{ .ErrorBudgetRatioMetric }}{{ .MetricFilter }} -`)) - -// Reference : https://github.com/slok/sloth/blob/2de193572284e36189fe78ab33beb7e2b339b0f8/internal/prometheus/alert_rules.go#L109 -// Multiburn multiwindow alert template. -var mwmbAlertTplBool = template.Must(template.New("mwmbAlertTpl").Option("missingkey=error").Parse(`(max({{ .QuickShortMetric }}{{ .MetricFilter}} > bool ({{ .QuickShortBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }}) and max({{ .QuickLongMetric }}{{ .MetricFilter}} > bool ({{ .QuickLongBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }})) or (max({{ .SlowShortMetric }}{{ .MetricFilter }} > bool ({{ .SlowShortBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }}) and max({{ .SlowQuickMetric }}{{ .MetricFilter }} > bool ({{ .SlowQuickBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }}))`)) -var mwmbAlertTpl = template.Must(template.New("mwmbAlertTpl").Option("missingkey=error").Parse(`(max({{ .QuickShortMetric }}{{ .MetricFilter}} > ({{ .QuickShortBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }}) and max({{ .QuickLongMetric }}{{ .MetricFilter}} > ({{ .QuickLongBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }})) or (max({{ .SlowShortMetric }}{{ .MetricFilter }} > ({{ .SlowShortBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }}) and max({{ .SlowQuickMetric }}{{ .MetricFilter }} > ({{ .SlowQuickBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }}))`)) - -// Pretty simple durations for prometheus. -func TimeDurationToPromStr(t time.Duration) string { - return prommodel.Duration(t).String() -} - -func MergeLabels(ms ...map[string]string) map[string]string { - res := map[string]string{} - for _, m := range ms { - for k, v := range m { - res[k] = v - } - } - return res -} - -func LeftJoinSlice[T comparable](arr1, arr2 []T) []T { - result := make([]T, len(arr1)) - cache := map[T]struct{}{} - for i, v := range arr1 { - cache[v] = struct{}{} - result[i] = v - } - for _, v := range arr2 { - if _, ok := cache[v]; !ok { - result = append(result, v) - } - } - return result -} - -func LeftJoinSliceAbstract[T any, S comparable](arr1, arr2 []T, getId func(T) S) []T { - result := make([]T, len(arr1)) - cache := map[S]struct{}{} - for i, v := range arr1 { - cache[getId(v)] = struct{}{} - result[i] = v - } - for _, v := range arr2 { - if _, ok := cache[getId(v)]; !ok { - result = append(result, v) - } - } - return result -} diff --git a/plugins/slo/test/test_plugin.go b/plugins/slo/test/test_plugin.go deleted file mode 100644 index 9eed321f90..0000000000 --- a/plugins/slo/test/test_plugin.go +++ /dev/null @@ -1,15 +0,0 @@ -package test - -import ( - "time" - - "github.com/rancher/opni/pkg/plugins/meta" - "github.com/rancher/opni/pkg/test" - sloapi "github.com/rancher/opni/plugins/slo/apis/slo" - "github.com/rancher/opni/plugins/slo/pkg/slo" -) - -func init() { - sloapi.MinEvaluateInterval = time.Second - test.EnablePlugin(meta.ModeGateway, slo.Scheme) -} diff --git a/test/integration/managementapi_cert_test.go b/test/integration/managementapi_cert_test.go deleted file mode 100644 index c06cce5bd3..0000000000 --- a/test/integration/managementapi_cert_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package integration_test - -import ( - "context" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "google.golang.org/protobuf/types/known/emptypb" - - managementv1 "github.com/rancher/opni/pkg/apis/management/v1" - "github.com/rancher/opni/pkg/test" - _ "github.com/rancher/opni/plugins/example/test" -) - -//#region Test Setup - -var _ = Describe("Management API Cerificate Management Tests", Ordered, Label("integration"), func() { - var environment *test.Environment - var client managementv1.ManagementClient - BeforeAll(func() { - environment = &test.Environment{} - Expect(environment.Start()).To(Succeed()) - DeferCleanup(environment.Stop) - client = environment.NewManagementClient() - }) - - //#endregion - - //#region Happy Path Tests - - It("can retrieve full certification chain information", func() { - certsInfo, err := client.CertsInfo(context.Background(), &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - - leaf := certsInfo.Chain[0] - root := certsInfo.Chain[len(certsInfo.Chain)-1] - - Expect(root.Issuer).To(Equal("CN=Example Root CA")) - Expect(root.Subject).To(Equal("CN=Example Root CA")) - Expect(root.IsCA).To(BeTrue()) - Expect(root.Fingerprint).NotTo(BeEmpty()) - - Expect(leaf.Issuer).To(Equal("CN=Example Root CA")) - Expect(leaf.Subject).To(Equal("CN=leaf")) - Expect(leaf.IsCA).To(BeFalse()) - Expect(leaf.Fingerprint).NotTo(BeEmpty()) - }) - - //#endregion - - //#region Edge Case Tests - - //TODO: Add Certificate Edge Case Tests - - //#endregion -}) diff --git a/test/plugins/alerting/output.log b/test/plugins/alerting/output.log new file mode 100644 index 0000000000..8f73deda13 --- /dev/null +++ b/test/plugins/alerting/output.log @@ -0,0 +1,20043 @@ +Running Suite: Alerting Suite - /home/ec2-user/opni/test/plugins/alerting +========================================================================= +Random Seed: 1694196603 + +Will run 11 of 54 specs +SSSSSSSSSSSSS•••• +------------------------------ +P [PENDING] +SLO Alerting when Managing SLOs with the mocked datasource when the SLOStore fails to create an SLO, the management server should not have it [integration] +/home/ec2-user/opni/test/plugins/alerting/slo_test.go:218 +------------------------------ +••• +------------------------------ +P [PENDING] +SLO Alerting when Managing SLOs with the mocked datasource should error when the fetching the status of non-existent SLO [integration] +/home/ec2-user/opni/test/plugins/alerting/slo_test.go:260 +------------------------------ +runtime: bad pointer in frame github.com/rancher/opni/test/plugins/alerting_test.glob..func6.2.10 at 0xc002d43ed0: 0x28 +fatal error: invalid pointer found on stack + +runtime stack: +runtime.throw({0x465c08a?, 0x755bb20?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/panic.go:1077 +0x5c fp=0x7f4223ffe838 sp=0x7f4223ffe808 pc=0x4748bc +runtime.adjustpointers(0x7f4223ffeab0?, 0x7f4223ffe8f8, 0x49cdc5?, {0x7f4223ffeab0?, 0x0?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/stack.go:627 +0x1ad fp=0x7f4223ffe898 sp=0x7f4223ffe838 pc=0x48fa0d +runtime.adjustframe(0x7f4223ffeab0, 0x7f4223ffe990) + /home/ec2-user/sdk/go1.21.0/src/runtime/stack.go:684 +0xdb fp=0x7f4223ffe928 sp=0x7f4223ffe898 pc=0x48fb3b +runtime.copystack(0xc002e956c0, 0x800000002?) + /home/ec2-user/sdk/go1.21.0/src/runtime/stack.go:935 +0x2c5 fp=0x7f4223ffec20 sp=0x7f4223ffe928 pc=0x4902e5 +runtime.newstack() + /home/ec2-user/sdk/go1.21.0/src/runtime/stack.go:1116 +0x47f fp=0x7f4223ffedd0 sp=0x7f4223ffec20 pc=0x49089f +runtime.morestack() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:593 +0x8f fp=0x7f4223ffedd8 sp=0x7f4223ffedd0 pc=0x4ab58f + +goroutine 1746 [copystack]: +runtime.checkptrBase(0xc00237e580) + /home/ec2-user/sdk/go1.21.0/src/runtime/checkptr.go:79 +0xd4 fp=0xc002d426c8 sp=0xc002d426c0 pc=0x441dd4 +runtime.checkptrArithmetic(0x0?, {0xc002d42720, 0x1, 0x4aead9?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/checkptr.go:58 +0x38 fp=0xc002d426f8 sp=0xc002d426c8 pc=0x441c38 +sync.indexLocal(0xc00237e400, 0x3) + /home/ec2-user/sdk/go1.21.0/src/sync/pool.go:280 +0x5c fp=0xc002d42738 sp=0xc002d426f8 pc=0x4bf2bc +sync.(*Pool).pin(0x770e520) + /home/ec2-user/sdk/go1.21.0/src/sync/pool.go:207 +0x4b fp=0xc002d42770 sp=0xc002d42738 pc=0x4beceb +sync.(*Pool).Get(0x770e520) + /home/ec2-user/sdk/go1.21.0/src/sync/pool.go:131 +0x35 fp=0xc002d427a8 sp=0xc002d42770 pc=0x4be975 +fmt.newPrinter() + /home/ec2-user/sdk/go1.21.0/src/fmt/print.go:152 +0x28 fp=0xc002d427d8 sp=0xc002d427a8 pc=0x591668 +fmt.Sprintf({0x4608983, 0x4}, {0xc002d42958, 0x1, 0x1}) + /home/ec2-user/sdk/go1.21.0/src/fmt/print.go:238 +0x3a fp=0xc002d42830 sp=0xc002d427d8 pc=0x591ffa +github.com/onsi/gomega/format.formatStruct({0x40ed880?, 0xc003195560?, 0x41a6d20?}, 0x5) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:478 +0x61d fp=0xc002d42a98 sp=0xc002d42830 pc=0xbaa91d +github.com/onsi/gomega/format.formatValue({0x40ed880, 0xc003195560, 0x1b9}, 0x5) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:384 +0xf2c fp=0xc002d42b80 sp=0xc002d42a98 pc=0xba8e2c +github.com/onsi/gomega/format.formatStruct({0x4315620?, 0xc003195560?, 0x41a6d20?}, 0x4) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:468 +0x22c fp=0xc002d42de8 sp=0xc002d42b80 pc=0xbaa52c +github.com/onsi/gomega/format.formatValue({0x4315620, 0xc003195560, 0x1b9}, 0x4) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:384 +0xf2c fp=0xc002d42ed0 sp=0xc002d42de8 pc=0xba8e2c +github.com/onsi/gomega/format.formatStruct({0x43aed20?, 0xc003195560?, 0x41a6d20?}, 0x3) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:468 +0x22c fp=0xc002d43138 sp=0xc002d42ed0 pc=0xbaa52c +github.com/onsi/gomega/format.formatValue({0x43aed20, 0xc003195560, 0x1b9}, 0x3) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:384 +0xf2c fp=0xc002d43220 sp=0xc002d43138 pc=0xba8e2c +github.com/onsi/gomega/format.formatValue({0x4330540, 0xc001f4eea8, 0x1b6}, 0x3) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:370 +0xcb4 fp=0xc002d43308 sp=0xc002d43220 pc=0xba8bb4 +github.com/onsi/gomega/format.formatStruct({0x4178aa0?, 0xc001f4eea8?, 0x41a6d20?}, 0x2) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:468 +0x22c fp=0xc002d43570 sp=0xc002d43308 pc=0xbaa52c +github.com/onsi/gomega/format.formatValue({0x4178aa0, 0xc001f4eea8, 0x1b9}, 0x2) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:384 +0xf2c fp=0xc002d43658 sp=0xc002d43570 pc=0xba8e2c +github.com/onsi/gomega/format.formatValue({0x42e29a0, 0xc00280eed0, 0x1b6}, 0x2) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:370 +0xcb4 fp=0xc002d43740 sp=0xc002d43658 pc=0xba8bb4 +github.com/onsi/gomega/format.formatStruct({0x4178b20?, 0xc00280eed0?, 0x41a6d20?}, 0x1) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:468 +0x22c fp=0xc002d439a8 sp=0xc002d43740 pc=0xbaa52c +github.com/onsi/gomega/format.formatValue({0x4178b20, 0xc00280eed0, 0x199}, 0x1) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:384 +0xf2c fp=0xc002d43a90 sp=0xc002d439a8 pc=0xba8e2c +github.com/onsi/gomega/format.formatValue({0x4178ba0, 0xc00280eed0, 0x16}, 0x1) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:370 +0xcb4 fp=0xc002d43b78 sp=0xc002d43a90 pc=0xba8bb4 +github.com/onsi/gomega/format.Object({0x4178ba0, 0xc00280eed0}, 0xc00280eed0?) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:265 +0x285 fp=0xc002d43cc0 sp=0xc002d43b78 pc=0xba72a5 +github.com/onsi/gomega/matchers.(*HaveOccurredMatcher).NegatedFailureMessage(0x7f42215d6dd0?, {0x4178ba0, 0xc00280eed0}) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/matchers/have_occurred_matcher.go:34 +0x3a fp=0xc002d43d28 sp=0xc002d43cc0 pc=0xc7f05a +github.com/onsi/gomega/internal.(*Assertion).match(0xc003079280, {0x5b29f98, 0x779c980}, 0x0, {0x0, 0x0, 0x0}) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/internal/assertion.go:103 +0x1d6 fp=0xc002d43df8 sp=0xc002d43d28 pc=0xbabc36 +github.com/onsi/gomega/internal.(*Assertion).NotTo(0xc003079280, {0x5b29f98, 0x779c980}, {0x0, 0x0, 0x0}) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/internal/assertion.go:74 +0x11e fp=0xc002d43e58 sp=0xc002d43df8 pc=0xbab85e +github.com/rancher/opni/test/plugins/alerting_test.glob..func6.2.10() + /home/ec2-user/opni/test/plugins/alerting/slo_test.go:273 +0x1bb fp=0xc002d43f00 sp=0xc002d43e58 pc=0x3dfd27b +github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func3({0x0, 0xc0030fecd0}) + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/node.go:463 +0x2f fp=0xc002d43f20 sp=0xc002d43f00 pc=0xb749cf +github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3() + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/suite.go:865 +0x106 fp=0xc002d43fe0 sp=0xc002d43f20 pc=0xb956c6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d43fe8 sp=0xc002d43fe0 pc=0x4ad421 +created by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 85 + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/suite.go:852 +0x1345 + +goroutine 1 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x60?, 0x78?, 0x18?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00211f6a8 sp=0xc00211f688 pc=0x4777ae +runtime.chanrecv(0xc000cb4150, 0xc000bd178f, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00211f720 sp=0xc00211f6a8 pc=0x441325 +runtime.chanrecv1(0x4552dc0?, 0x3f99540?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00211f748 sp=0xc00211f720 pc=0x440f72 +testing.(*T).Run(0xc000e4b1e0, {0x4620aba, 0xc}, 0x47eae88) + /home/ec2-user/sdk/go1.21.0/src/testing/testing.go:1649 +0x856 fp=0xc00211f868 sp=0xc00211f748 pc=0x5ef6d6 +testing.runTests.func1(0x0?) + /home/ec2-user/sdk/go1.21.0/src/testing/testing.go:2054 +0x85 fp=0xc00211f8c0 sp=0xc00211f868 pc=0x5f3205 +testing.tRunner(0xc000e4b1e0, 0xc000bd1b08) + /home/ec2-user/sdk/go1.21.0/src/testing/testing.go:1595 +0x239 fp=0xc00211f9d8 sp=0xc00211f8c0 pc=0x5ede59 +testing.runTests(0xc0005ab0e0?, {0x7702460, 0x1, 0x1}, {0x1c?, 0x4aead9?, 0x0?}) + /home/ec2-user/sdk/go1.21.0/src/testing/testing.go:2052 +0x897 fp=0xc00211fb38 sp=0xc00211f9d8 pc=0x5f3077 +testing.(*M).Run(0xc0005ab0e0) + /home/ec2-user/sdk/go1.21.0/src/testing/testing.go:1925 +0xb58 fp=0xc00211feb8 sp=0xc00211fb38 pc=0x5f0898 +main.main() + _testmain.go:47 +0x2be fp=0xc00211ff40 sp=0xc00211feb8 pc=0x3e0c3fe +runtime.main() + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:267 +0x2bb fp=0xc00211ffe0 sp=0xc00211ff40 pc=0x47733b +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00211ffe8 sp=0xc00211ffe0 pc=0x4ad421 + +goroutine 2 [force gc (idle)]: +runtime.gopark(0x7541700?, 0x776b4c0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0000847a8 sp=0xc000084788 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +runtime.forcegchelper() + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:322 +0xb3 fp=0xc0000847e0 sp=0xc0000847a8 pc=0x477613 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0000847e8 sp=0xc0000847e0 pc=0x4ad421 +created by runtime.init.6 in goroutine 1 + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:310 +0x1a + +goroutine 3 [GC sweep wait]: +runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000096f78 sp=0xc000096f58 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +runtime.bgsweep(0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/mgcsweep.go:321 +0xdf fp=0xc000096fc8 sp=0xc000096f78 pc=0x4614ff +runtime.gcenable.func1() + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:200 +0x25 fp=0xc000096fe0 sp=0xc000096fc8 pc=0x4565e5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000096fe8 sp=0xc000096fe0 pc=0x4ad421 +created by runtime.gcenable in goroutine 1 + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:200 +0x66 + +goroutine 4 [GC scavenge wait]: +runtime.gopark(0xc00005a0e0?, 0x5afd9a0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000090f70 sp=0xc000090f50 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +runtime.(*scavengerState).park(0x7769d40) + /home/ec2-user/sdk/go1.21.0/src/runtime/mgcscavenge.go:425 +0x49 fp=0xc000090fa0 sp=0xc000090f70 pc=0x45ece9 +runtime.bgscavenge(0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/mgcscavenge.go:658 +0x59 fp=0xc000090fc8 sp=0xc000090fa0 pc=0x45f279 +runtime.gcenable.func2() + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:201 +0x25 fp=0xc000090fe0 sp=0xc000090fc8 pc=0x456585 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000090fe8 sp=0xc000090fe0 pc=0x4ad421 +created by runtime.gcenable in goroutine 1 + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:201 +0xa5 + +goroutine 5 [finalizer wait]: +runtime.gopark(0x0?, 0x47ef678?, 0x80?, 0x3?, 0x1000000010?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f0be28 sp=0xc001f0be08 pc=0x4777ae +runtime.runfinq() + /home/ec2-user/sdk/go1.21.0/src/runtime/mfinal.go:193 +0x13b fp=0xc001f0bfe0 sp=0xc001f0be28 pc=0x45565b +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f0bfe8 sp=0xc001f0bfe0 pc=0x4ad421 +created by runtime.createfing in goroutine 1 + /home/ec2-user/sdk/go1.21.0/src/runtime/mfinal.go:163 +0x3d + +goroutine 6 [GC worker (idle)]: +runtime.gopark(0x7c5b0aa7766c0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000183f50 sp=0xc000183f30 pc=0x4777ae +runtime.gcBgMarkWorker() + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:1293 +0xe5 fp=0xc000183fe0 sp=0xc000183f50 pc=0x458165 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000183fe8 sp=0xc000183fe0 pc=0x4ad421 +created by runtime.gcBgMarkStartWorkers in goroutine 1 + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:1217 +0x1c + +goroutine 18 [GC worker (idle)]: +runtime.gopark(0x7c5b0a074d332?, 0x3?, 0x44?, 0xc8?, 0xc000084fb8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000182f50 sp=0xc000182f30 pc=0x4777ae +runtime.gcBgMarkWorker() + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:1293 +0xe5 fp=0xc000182fe0 sp=0xc000182f50 pc=0x458165 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000182fe8 sp=0xc000182fe0 pc=0x4ad421 +created by runtime.gcBgMarkStartWorkers in goroutine 1 + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:1217 +0x1c + +goroutine 34 [GC worker (idle)]: +runtime.gopark(0x7c5b0a0746a3e?, 0x3?, 0xd4?, 0x16?, 0xc000085790?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000181f50 sp=0xc000181f30 pc=0x4777ae +runtime.gcBgMarkWorker() + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:1293 +0xe5 fp=0xc000181fe0 sp=0xc000181f50 pc=0x458165 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000181fe8 sp=0xc000181fe0 pc=0x4ad421 +created by runtime.gcBgMarkStartWorkers in goroutine 1 + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:1217 +0x1c + +goroutine 7 [GC worker (idle)]: +runtime.gopark(0x77a02c0?, 0x1?, 0x5c?, 0xc5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000180f50 sp=0xc000180f30 pc=0x4777ae +runtime.gcBgMarkWorker() + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:1293 +0xe5 fp=0xc000180fe0 sp=0xc000180f50 pc=0x458165 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000180fe8 sp=0xc000180fe0 pc=0x4ad421 +created by runtime.gcBgMarkStartWorkers in goroutine 1 + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:1217 +0x1c + +goroutine 8 [select]: +runtime.gopark(0xc000092f90?, 0x2?, 0x0?, 0x0?, 0xc000092f0c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000092d80 sp=0xc000092d60 pc=0x4777ae +runtime.selectgo(0xc000092f90, 0xc000092f08, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000092ed0 sp=0xc000092d80 pc=0x48916b +github.com/golang/glog.(*fileSink).flushDaemon(0x7769918) + /home/ec2-user/go/pkg/mod/github.com/golang/glog@v1.1.1/glog_file.go:346 +0x15c fp=0xc000092fc0 sp=0xc000092ed0 pc=0x20e479c +github.com/golang/glog.init.1.func1() + /home/ec2-user/go/pkg/mod/github.com/golang/glog@v1.1.1/glog_file.go:161 +0x34 fp=0xc000092fe0 sp=0xc000092fc0 pc=0x20e2f54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000092fe8 sp=0xc000092fe0 pc=0x4ad421 +created by github.com/golang/glog.init.1 in goroutine 1 + /home/ec2-user/go/pkg/mod/github.com/golang/glog@v1.1.1/glog_file.go:161 +0x225 + +goroutine 9 [select]: +runtime.gopark(0xc000093f80?, 0x3?, 0x0?, 0x0?, 0xc000093f1a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000093d90 sp=0xc000093d70 pc=0x4777ae +runtime.selectgo(0xc000093f80, 0xc000093f14, 0xc000436680?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000093ee0 sp=0xc000093d90 pc=0x48916b +go.opencensus.io/stats/view.(*worker).start(0xc000436680) + /home/ec2-user/go/pkg/mod/go.opencensus.io@v0.24.0/stats/view/worker.go:292 +0x199 fp=0xc000093fc0 sp=0xc000093ee0 pc=0x3bf4f39 +go.opencensus.io/stats/view.init.0.func1() + /home/ec2-user/go/pkg/mod/go.opencensus.io@v0.24.0/stats/view/worker.go:34 +0x34 fp=0xc000093fe0 sp=0xc000093fc0 pc=0x3bf39b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000093fe8 sp=0xc000093fe0 pc=0x4ad421 +created by go.opencensus.io/stats/view.init.0 in goroutine 1 + /home/ec2-user/go/pkg/mod/go.opencensus.io@v0.24.0/stats/view/worker.go:34 +0xfd + +goroutine 117 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0xf0?, 0xa1?, 0xc000187f98?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000187f08 sp=0xc000187ee8 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc000187f80 sp=0xc000187f08 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0x7f429a401500?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc000187fa8 sp=0xc000187f80 pc=0x440f72 +github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore.func3() + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:99 +0x4a fp=0xc000187fe0 sp=0xc000187fa8 pc=0x24a06aa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000187fe8 sp=0xc000187fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore in goroutine 65 + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:98 +0x4cb + +goroutine 87 [IO wait]: +runtime.gopark(0x4?, 0xc000091c30?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000091c48 sp=0xc000091c28 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000091c80 sp=0xc000091c48 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6eee0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000091ca0 sp=0xc000091c80 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000dd7100, 0xc000de2000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000091ce8 sp=0xc000091ca0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000dd70e0, {0xc000de2000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc000091dd8 sp=0xc000091ce8 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc00007d670, {0xc000de2000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc000091e38 sp=0xc000091dd8 pc=0x51ed0d +io.copyBuffer({0x5b0ede0, 0xc0008621b0}, {0x5b0eb40, 0xc00007d670}, {0x0, 0x0, 0x0}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:430 +0x27e fp=0xc000091ee0 sp=0xc000091e38 pc=0x50115e +io.Copy(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:389 +os/exec.(*Cmd).writerDescriptor.func1() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:560 +0x5b fp=0xc000091f50 sp=0xc000091ee0 pc=0x6552db +os/exec.(*Cmd).Start.func2(0xc000996600) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:717 +0x3d fp=0xc000091fb8 sp=0xc000091f50 pc=0x65669d +os/exec.(*Cmd).Start.func3() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:729 +0x42 fp=0xc000091fe0 sp=0xc000091fb8 pc=0x656622 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000091fe8 sp=0xc000091fe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 65 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:716 +0xf3a + +goroutine 116 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc0007d2640?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0008b4f08 sp=0xc0008b4ee8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc000ef4250, 0x0) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc0008b4f58 sp=0xc0008b4f08 pc=0x4a9339 +sync.(*Cond).Wait(0xc000ef4240) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc0008b4f88 sp=0xc0008b4f58 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*asyncCallbacksHandler).asyncCBDispatcher(0xc0007d2640) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2842 +0x153 fp=0xc0008b4fc0 sp=0xc0008b4f88 pc=0x1f759b3 +github.com/nats-io/nats%2ego.Options.Connect.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x34 fp=0xc0008b4fe0 sp=0xc0008b4fc0 pc=0x1f6a534 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0008b4fe8 sp=0xc0008b4fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.Options.Connect in goroutine 65 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x86b + +goroutine 86 [select, locked to thread]: +runtime.gopark(0xc000095fa8?, 0x2?, 0x0?, 0x0?, 0xc000095fa4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000095e08 sp=0xc000095de8 pc=0x4777ae +runtime.selectgo(0xc000095fa8, 0xc000095fa0, 0x0?, 0x0, 0xc0008ab150?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000095f58 sp=0xc000095e08 pc=0x48916b +runtime.ensureSigM.func1() + /home/ec2-user/sdk/go1.21.0/src/runtime/signal_unix.go:1014 +0x19f fp=0xc000095fe0 sp=0xc000095f58 pc=0x4a35df +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000095fe8 sp=0xc000095fe0 pc=0x4ad421 +created by runtime.ensureSigM in goroutine 85 + /home/ec2-user/sdk/go1.21.0/src/runtime/signal_unix.go:997 +0xc8 + +goroutine 85 [select]: +runtime.gopark(0xc00105f408?, 0x5?, 0x10?, 0xea?, 0xc00105eefe?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00105e9c0 sp=0xc00105e9a0 pc=0x4777ae +runtime.selectgo(0xc00105f408, 0xc00105eef4, 0x7769700?, 0x0, 0x710b428a?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00105eb10 sp=0xc00105e9c0 pc=0x48916b +github.com/onsi/ginkgo/v2/internal.(*Suite).runNode(_, {0x39, 0x4, {0x468d28e, 0x29}, 0xc0001255c0, {{0x64228e0, 0x35}, 0x10d, {0x0, ...}, ...}, ...}, ...) + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/suite.go:887 +0x182f fp=0xc001062a40 sp=0xc00105eb10 pc=0xb9372f +github.com/onsi/ginkgo/v2/internal.(*group).attemptSpec(0xc001066830, 0x1, {{0xc000f9a480?, 0xc0030d2ee0?, 0x2?}, 0x0?}) + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/group.go:199 +0x1125 fp=0xc001064ab0 sp=0xc001062a40 pc=0xb68705 +github.com/onsi/ginkgo/v2/internal.(*group).run(0xc001066830, {0xc0017df6c0, 0xd, 0xd}) + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/group.go:349 +0x1228 fp=0xc001066498 sp=0xc001064ab0 pc=0xb6c0e8 +github.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000ca4000, {0x462619f, 0xe}, {0x779c980, 0x0, 0x0}, {0xc000060064, 0x29}, 0x0, {0xc000ac3800, ...}) + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/suite.go:465 +0x1167 fp=0xc001067270 sp=0xc001066498 pc=0xb8bc07 +github.com/onsi/ginkgo/v2/internal.(*Suite).Run(_, {_, _}, {_, _, _}, {_, _}, _, {0x5b4ff78, ...}, ...) + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/suite.go:116 +0x5f8 fp=0xc001067438 sp=0xc001067270 pc=0xb85518 +github.com/onsi/ginkgo/v2.RunSpecs({0x5b0ed20, 0xc000e4b380}, {0x462619f, 0xe}, {0x0, 0x0, 0x0?}) + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/core_dsl.go:319 +0x166b fp=0xc001067e48 sp=0xc001067438 pc=0xba34cb +github.com/rancher/opni/test/plugins/alerting_test.TestAlerting(0x0?) + /home/ec2-user/opni/test/plugins/alerting/alerting_suite_test.go:40 +0x4e fp=0xc001067e98 sp=0xc001067e48 pc=0x3dc86ee +testing.tRunner(0xc000e4b380, 0x47eae88) + /home/ec2-user/sdk/go1.21.0/src/testing/testing.go:1595 +0x239 fp=0xc001067fb0 sp=0xc001067e98 pc=0x5ede59 +testing.(*T).Run.func1() + /home/ec2-user/sdk/go1.21.0/src/testing/testing.go:1648 +0x45 fp=0xc001067fe0 sp=0xc001067fb0 pc=0x5ef945 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001067fe8 sp=0xc001067fe0 pc=0x4ad421 +created by testing.(*T).Run in goroutine 1 + /home/ec2-user/sdk/go1.21.0/src/testing/testing.go:1648 +0x82b + +goroutine 62 [syscall]: +runtime.notetsleepg(0x4b0571?, 0x4ad421?) + /home/ec2-user/sdk/go1.21.0/src/runtime/lock_futex.go:236 +0x29 fp=0xc000084fa0 sp=0xc000084f68 pc=0x447049 +os/signal.signal_recv() + /home/ec2-user/sdk/go1.21.0/src/runtime/sigqueue.go:152 +0x29 fp=0xc000084fc0 sp=0xc000084fa0 pc=0x4a9729 +os/signal.loop() + /home/ec2-user/sdk/go1.21.0/src/os/signal/signal_unix.go:23 +0x1d fp=0xc000084fe0 sp=0xc000084fc0 pc=0x6a909d +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000084fe8 sp=0xc000084fe0 pc=0x4ad421 +created by os/signal.Notify.func1.1 in goroutine 85 + /home/ec2-user/sdk/go1.21.0/src/os/signal/signal.go:151 +0x47 + +goroutine 63 [select]: +runtime.gopark(0xc000185f78?, 0x3?, 0x0?, 0x0?, 0xc000185f22?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000185d88 sp=0xc000185d68 pc=0x4777ae +runtime.selectgo(0xc000185f78, 0xc000185f1c, 0xc000185f28?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000185ed8 sp=0xc000185d88 pc=0x48916b +github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0x0) + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/interrupt_handler/interrupt_handler.go:131 +0x125 fp=0xc000185fb8 sp=0xc000185ed8 pc=0xb57045 +github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func3() + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/interrupt_handler/interrupt_handler.go:158 +0x42 fp=0xc000185fe0 sp=0xc000185fb8 pc=0xb56ee2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000185fe8 sp=0xc000185fe0 pc=0x4ad421 +created by github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 85 + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/interrupt_handler/interrupt_handler.go:128 +0x2bd + +goroutine 64 [select]: +runtime.gopark(0xc001087fb0?, 0x2?, 0xff?, 0xff?, 0xc001087f7c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001087df0 sp=0xc001087dd0 pc=0x4777ae +runtime.selectgo(0xc001087fb0, 0xc001087f78, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001087f40 sp=0xc001087df0 pc=0x48916b +github.com/onsi/ginkgo/v2/internal.RegisterForProgressSignal.func1() + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/progress_report.go:32 +0xc7 fp=0xc001087fe0 sp=0xc001087f40 pc=0xb7f707 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001087fe8 sp=0xc001087fe0 pc=0x4ad421 +created by github.com/onsi/ginkgo/v2/internal.RegisterForProgressSignal in goroutine 85 + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/progress_report.go:30 +0x189 + +goroutine 636 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f419a8 sp=0xc001f41988 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001f419e0 sp=0xc001f419a8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163b298, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001f41a00 sp=0xc001f419e0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001c9bc20, 0x448801?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001f41a48 sp=0xc001f41a00 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc001c9bc00) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001f41b38 sp=0xc001f41a48 pc=0x513f65 +net.(*netFD).accept(0xc001c9bc00) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001f41c30 sp=0xc001f41b38 pc=0x8967be +net.(*TCPListener).accept(0xc0007d2a60) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc001f41c78 sp=0xc001f41c30 pc=0x8c383e +net.(*TCPListener).Accept(0xc0007d2a60) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc001f41ce8 sp=0xc001f41c78 pc=0x8c1ba5 +net/http.(*onceCloseListener).Accept(0xc000e38810) + :1 +0x44 fp=0xc001f41d20 sp=0xc001f41ce8 pc=0xab5544 +net/http.(*Server).Serve(0xc001ec2780, {0x5b2a6d0, 0xc0007d2a60}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc001f41f18 sp=0xc001f41d20 pc=0xa7859f +net/http.(*Server).ListenAndServe(0xc001ec2780) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:2985 +0xbd fp=0xc001f41f70 sp=0xc001f41f18 pc=0xa77f9d +github.com/rancher/opni/pkg/management.(*Server).listenAndServeHttp.func2() + /home/ec2-user/opni/pkg/management/server.go:246 +0x2f fp=0xc001f41fa0 sp=0xc001f41f70 pc=0x231edef +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc001f41fe0 sp=0xc001f41fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f41fe8 sp=0xc001f41fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 496 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 88 [IO wait]: +runtime.gopark(0x4b05a9?, 0xc0008aec70?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0008aec48 sp=0xc0008aec28 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0008aec80 sp=0xc0008aec48 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6ecf0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0008aeca0 sp=0xc0008aec80 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000dd71c0, 0xc000128000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0008aece8 sp=0xc0008aeca0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000dd71a0, {0xc000128000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0008aedd8 sp=0xc0008aece8 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc00007d688, {0xc000128000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc0008aee38 sp=0xc0008aedd8 pc=0x51ed0d +io.copyBuffer({0x5b0ede0, 0xc0008621c8}, {0x5b0eb40, 0xc00007d688}, {0x0, 0x0, 0x0}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:430 +0x27e fp=0xc0008aeee0 sp=0xc0008aee38 pc=0x50115e +io.Copy(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:389 +os/exec.(*Cmd).writerDescriptor.func1() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:560 +0x5b fp=0xc0008aef50 sp=0xc0008aeee0 pc=0x6552db +os/exec.(*Cmd).Start.func2(0xc000996640) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:717 +0x3d fp=0xc0008aefb8 sp=0xc0008aef50 pc=0x65669d +os/exec.(*Cmd).Start.func3() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:729 +0x42 fp=0xc0008aefe0 sp=0xc0008aefb8 pc=0x656622 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0008aefe8 sp=0xc0008aefe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 65 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:716 +0xf3a + +goroutine 89 [select]: +runtime.gopark(0xc0009822b0?, 0x656a30?, 0x0?, 0x1?, 0xc0008b2e7c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0008b2ce0 sp=0xc0008b2cc0 pc=0x4777ae +runtime.selectgo(0xc0008b2f60, 0xc0008b2e78, 0xc0009b9168?, 0x1, 0xc0009c5bd0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0008b2e30 sp=0xc0008b2ce0 pc=0x48916b +os/exec.(*Cmd).watchCtx(0xc000b32000, 0xc0009822a0) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:757 +0x10f fp=0xc0008b2fb0 sp=0xc0008b2e30 pc=0x656a6f +os/exec.(*Cmd).Start.func4() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:743 +0x45 fp=0xc0008b2fe0 sp=0xc0008b2fb0 pc=0x6565a5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0008b2fe8 sp=0xc0008b2fe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 65 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:743 +0xf05 + +goroutine 90 [syscall]: +syscall.Syscall6(0x8200000000?, 0x82000000c7?, 0x20000000c5?, 0x1ef?, 0x64?, 0x1ef?, 0x63?) + /home/ec2-user/sdk/go1.21.0/src/syscall/syscall_linux.go:91 +0x3d fp=0xc0008b3d38 sp=0xc0008b3cb0 pc=0x4d1b1d +os.(*Process).blockUntilWaitable(0xc0009a8f00) + /home/ec2-user/sdk/go1.21.0/src/os/wait_waitid.go:32 +0x9f fp=0xc0008b3e18 sp=0xc0008b3d38 pc=0x529c5f +os.(*Process).wait(0xc0009a8f00) + /home/ec2-user/sdk/go1.21.0/src/os/exec_unix.go:22 +0x4b fp=0xc0008b3e98 sp=0xc0008b3e18 pc=0x51e0ab +os.(*Process).Wait(...) + /home/ec2-user/sdk/go1.21.0/src/os/exec.go:134 +os/exec.(*Cmd).Wait(0xc000b32000) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:890 +0xb9 fp=0xc0008b3f60 sp=0xc0008b3e98 pc=0x657279 +github.com/onsi/gomega/gexec.(*Session).monitorForExit(0xc000ddd140, 0x5b21760?) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:196 +0x3f fp=0xc0008b3fb0 sp=0xc0008b3f60 pc=0x233731f +github.com/onsi/gomega/gexec.Start.func1() + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:96 +0x45 fp=0xc0008b3fe0 sp=0xc0008b3fb0 pc=0x2336d65 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0008b3fe8 sp=0xc0008b3fe0 pc=0x4ad421 +created by github.com/onsi/gomega/gexec.Start in goroutine 65 + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:96 +0x9eb + +goroutine 91 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x8c?, 0xed?, 0x446b50?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001084e78 sp=0xc001084e58 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001084ef0 sp=0xc001084e78 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0xc000085f80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001084f18 sp=0xc001084ef0 pc=0x440f72 +github.com/rancher/opni/pkg/test.(*Environment).startJetstream.func1() + /home/ec2-user/opni/pkg/test/environment.go:562 +0x69 fp=0xc001084f50 sp=0xc001084f18 pc=0x24b23e9 +github.com/rancher/opni/pkg/util/waitctx.restrictive.Go.func1() + /home/ec2-user/opni/pkg/util/waitctx/context.go:162 +0x93 fp=0xc001084fe0 sp=0xc001084f50 pc=0x178df73 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001084fe8 sp=0xc001084fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/util/waitctx.restrictive.Go in goroutine 65 + /home/ec2-user/opni/pkg/util/waitctx/context.go:160 +0xeb + +goroutine 100 [select]: +runtime.gopark(0xc000d03f48?, 0x2?, 0xb2?, 0x0?, 0xc000d03f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d03d80 sp=0xc000d03d60 pc=0x4777ae +runtime.selectgo(0xc000d03f48, 0xc000d03f00, 0x104000000b2?, 0x0, 0x144000000b2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d03ed0 sp=0xc000d03d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).Recv(0xc000fe6b80) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:234 +0xeb fp=0xc000d03f78 sp=0xc000d03ed0 pc=0x2155d2b +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc000ee4410) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc000d03fc0 sp=0xc000d03f78 pc=0x215759f +github.com/hashicorp/go-plugin.newGRPCClient.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x34 fp=0xc000d03fe0 sp=0xc000d03fc0 pc=0x21599b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d03fe8 sp=0xc000d03fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 154 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x5b7 + +goroutine 115 [chan receive]: +runtime.gopark(0x4aef8c?, 0x4b0a55?, 0x28?, 0x71?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001081e18 sp=0xc001081df8 pc=0x4777ae +runtime.chanrecv(0xc000066480, 0xc001081ed0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001081e90 sp=0xc001081e18 pc=0x441325 +runtime.chanrecv2(0xc000c77128?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc001081eb8 sp=0xc001081e90 pc=0x440f92 +github.com/nats-io/nats%2ego.(*Conn).flusher(0xc000c77100) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:3324 +0x1a5 fp=0xc001081fc0 sp=0xc001081eb8 pc=0x1f792e5 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x34 fp=0xc001081fe0 sp=0xc001081fc0 pc=0x1f714b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001081fe8 sp=0xc001081fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 65 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x4cb + +goroutine 114 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x26?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001080bd0 sp=0xc001080bb0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001080c08 sp=0xc001080bd0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6ebf8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001080c28 sp=0xc001080c08 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000436220, 0xc000df6000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001080c70 sp=0xc001080c28 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000436200, {0xc000df6000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001080d60 sp=0xc001080c70 pc=0x50e5a5 +net.(*netFD).Read(0xc000436200, {0xc000df6000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001080dc0 sp=0xc001080d60 pc=0x89342b +net.(*conn).Read(0xc000b780d8, {0xc000df6000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001080e48 sp=0xc001080dc0 pc=0x8b176d +net.(*TCPConn).Read(0xc000b780d8, {0xc000df6000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc001080e90 sp=0xc001080e48 pc=0x8d72f4 +github.com/nats-io/nats%2ego.(*natsReader).Read(0xc000ef4b00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1890 +0x1a9 fp=0xc001080f10 sp=0xc001080e90 pc=0x1f6d169 +github.com/nats-io/nats%2ego.(*Conn).readLoop(0xc000c77100) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2917 +0x1f8 fp=0xc001080fc0 sp=0xc001080f10 pc=0x1f75fb8 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x34 fp=0xc001080fe0 sp=0xc001080fc0 pc=0x1f71514 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001080fe8 sp=0xc001080fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 65 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x44e + +goroutine 96 [chan receive]: +runtime.gopark(0x0?, 0x10?, 0xa0?, 0xf0?, 0xc0027a51e4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001082d10 sp=0xc001082cf0 pc=0x4777ae +runtime.chanrecv(0xc0008abe00, 0xc001082e40, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001082d88 sp=0xc001082d10 pc=0x441325 +runtime.chanrecv2(0x3b9aca00?, 0x7f429a406300?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc001082db0 sp=0xc001082d88 pc=0x440f92 +github.com/hashicorp/go-plugin.pidWait(0xc000d74200?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/process.go:20 +0xdb fp=0xc001082e68 sp=0xc001082db0 pc=0x216033b +github.com/hashicorp/go-plugin.(*Client).reattach.func1(0x0?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:853 +0xd8 fp=0xc001082fb8 sp=0xc001082e68 pc=0x21528b8 +github.com/hashicorp/go-plugin.(*Client).reattach.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:862 +0x42 fp=0xc001082fe0 sp=0xc001082fb8 pc=0x21527a2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001082fe8 sp=0xc001082fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*Client).reattach in goroutine 154 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:846 +0x365 + +goroutine 118 [select]: +runtime.gopark(0xc000066730?, 0x13b0513?, 0x0?, 0x1?, 0xc0008afecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0008afd30 sp=0xc0008afd10 pc=0x4777ae +runtime.selectgo(0xc0008aff60, 0xc0008afec8, 0x446b50?, 0x1, 0xc000405798?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0008afe80 sp=0xc0008afd30 pc=0x48916b +github.com/lestrrat-go/backoff/v2.(*controller).loop(0xc000c95860) + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:64 +0x318 fp=0xc0008affc0 sp=0xc0008afe80 pc=0x13b0538 +github.com/lestrrat-go/backoff/v2.newController.func1() + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x34 fp=0xc0008affe0 sp=0xc0008affc0 pc=0x13b01f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0008affe8 sp=0xc0008affe0 pc=0x4ad421 +created by github.com/lestrrat-go/backoff/v2.newController in goroutine 65 + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x4dd + +goroutine 28 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc000ac80d0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f6ddd8 sp=0xc000f6ddb8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc0003f8410, 0x19) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc000f6de28 sp=0xc000f6ddd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc0003f8400) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc000f6de58 sp=0xc000f6de28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc000c77100, 0xc000ac80d0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc000f6dfb0 sp=0xc000f6de58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc000f6dfe0 sp=0xc000f6dfb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f6dfe8 sp=0xc000f6dfe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 65 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 125 [select]: +runtime.gopark(0xc0008b5e40?, 0x3?, 0xb4?, 0x1b?, 0xc0008b5c2a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0008b5a48 sp=0xc0008b5a28 pc=0x4777ae +runtime.selectgo(0xc0008b5e40, 0xc0008b5c24, 0x0?, 0x0, 0xc0009e77c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0008b5b98 sp=0xc0008b5a48 pc=0x48916b +github.com/rancher/opni/pkg/health.(*Monitor).Run(0xc00106e820, {0x5b426e0, 0xc000b69f40}, {0x5b21aa8, 0xc000436580}) + /home/ec2-user/opni/pkg/health/monitor.go:69 +0x1f3 fp=0xc0008b5f80 sp=0xc0008b5b98 pc=0x206e173 +github.com/rancher/opni/pkg/gateway.NewGateway.func23() + /home/ec2-user/opni/pkg/gateway/gateway.go:277 +0x6c fp=0xc0008b5fe0 sp=0xc0008b5f80 pc=0x22aed8c +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0008b5fe8 sp=0xc0008b5fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.NewGateway in goroutine 65 + /home/ec2-user/opni/pkg/gateway/gateway.go:277 +0x3eed + +goroutine 596 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020e19e0 sp=0xc0020e19c0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0020e1a18 sp=0xc0020e19e0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a870, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0020e1a38 sp=0xc0020e1a18 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0019e7e20, 0x4b0501?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0020e1a80 sp=0xc0020e1a38 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0019e7e00) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0020e1b70 sp=0xc0020e1a80 pc=0x513f65 +net.(*netFD).accept(0xc0019e7e00) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0020e1c68 sp=0xc0020e1b70 pc=0x8967be +net.(*TCPListener).accept(0xc001f0e380) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc0020e1cb0 sp=0xc0020e1c68 pc=0x8c383e +net.(*TCPListener).Accept(0xc001f0e380) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc0020e1d20 sp=0xc0020e1cb0 pc=0x8c1ba5 +google.golang.org/grpc.(*Server).Serve(0xc000e0c3c0, {0x5b2a6d0, 0xc001f0e380}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc0020e1f50 sp=0xc0020e1d20 pc=0x117789e +github.com/rancher/opni/pkg/management.(*Server).listenAndServeGrpc.func1() + /home/ec2-user/opni/pkg/management/server.go:213 +0x5e fp=0xc0020e1fa0 sp=0xc0020e1f50 pc=0x231e69e +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0020e1fe0 sp=0xc0020e1fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020e1fe8 sp=0xc0020e1fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 495 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 126 [chan receive]: +runtime.gopark(0x0?, 0x4b05a9?, 0x50?, 0x5e?, 0x4bff4a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001085e18 sp=0xc001085df8 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001085e90 sp=0xc001085e18 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0x7f429a405700?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001085eb8 sp=0xc001085e90 pc=0x440f72 +github.com/rancher/opni/pkg/gateway.NewGateway.func5() + /home/ec2-user/opni/pkg/gateway/gateway.go:308 +0x5c fp=0xc001085f50 sp=0xc001085eb8 pc=0x22aea9c +github.com/rancher/opni/pkg/util/waitctx.restrictive.Go.func1() + /home/ec2-user/opni/pkg/util/waitctx/context.go:162 +0x93 fp=0xc001085fe0 sp=0xc001085f50 pc=0x178df73 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001085fe8 sp=0xc001085fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/util/waitctx.restrictive.Go in goroutine 65 + /home/ec2-user/opni/pkg/util/waitctx/context.go:160 +0xeb + +goroutine 553 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bc48d8 sp=0xc001bc48b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001bc4910 sp=0xc001bc48d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677bc8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001bc4930 sp=0xc001bc4910 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d73920, 0xc001de8601?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001bc4978 sp=0xc001bc4930 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc000d73900) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001bc4a68 sp=0xc001bc4978 pc=0x513f65 +net.(*netFD).accept(0xc000d73900) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001bc4b60 sp=0xc001bc4a68 pc=0x8967be +net.(*TCPListener).accept(0xc000997980) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc001bc4ba8 sp=0xc001bc4b60 pc=0x8c383e +net.(*TCPListener).Accept(0xc000997980) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc001bc4c18 sp=0xc001bc4ba8 pc=0x8c1ba5 +crypto/tls.(*listener).Accept(0xc000863bd8) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/tls.go:66 +0x43 fp=0xc001bc4c70 sp=0xc001bc4c18 pc=0x9b2d43 +net/http.(*onceCloseListener).Accept(0xc001dcf740) + :1 +0x44 fp=0xc001bc4ca8 sp=0xc001bc4c70 pc=0xab5544 +net/http.(*Server).Serve(0xc001da51d0, {0x5b2a5b0, 0xc000863bd8}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc001bc4ea0 sp=0xc001bc4ca8 pc=0xa7859f +net/http.Serve(...) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:2595 +github.com/gin-gonic/gin.(*Engine).RunListener(0xc000c4c820, {0x5b2a5b0, 0xc000863bd8}) + /home/ec2-user/go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/gin.go:565 +0x370 fp=0xc001bc4f90 sp=0xc001bc4ea0 pc=0x171e870 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure.func1() + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:72 +0x5c fp=0xc001bc4fe0 sp=0xc001bc4f90 pc=0x262a57c +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bc4fe8 sp=0xc001bc4fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure in goroutine 176 + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:71 +0x26d + +goroutine 573 [select]: +runtime.gopark(0xc001f09478?, 0x2?, 0xe8?, 0x28?, 0xc001f09414?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f09280 sp=0xc001f09260 pc=0x4777ae +runtime.selectgo(0xc001f09478, 0xc001f09410, 0xc001dcf980?, 0x0, 0xc001f09470?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001f093d0 sp=0xc001f09280 pc=0x48916b +github.com/rancher/opni/pkg/management.(*Server).WatchClusterHealthStatus(0xc000d8a000, 0x42219c0?, {0x5b4cae0, 0xc001dc4ed0}) + /home/ec2-user/opni/pkg/management/health.go:38 +0x352 fp=0xc001f094a8 sp=0xc001f093d0 pc=0x23189f2 +github.com/rancher/opni/pkg/apis/management/v1._Management_WatchClusterHealthStatus_Handler({0x4537040?, 0xc000d8a000}, {0x5b4ab30, 0xc001dcf9b0}) + /home/ec2-user/opni/pkg/apis/management/v1/management_grpc.pb.go:829 +0xfa fp=0xc001f09508 sp=0xc001f094a8 pc=0x1781ada +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4537040, 0xc000d8a000}, {0x5b492f8, 0xc001da52c0}, 0xc00149ffb0, 0x47ec6b8) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001f09848 sp=0xc001f09508 pc=0x12b0e02 +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef860, {0x5b54e00, 0xc001174680}, 0xc001f18240, 0xc000ff8960, 0x7716c80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001f09db0 sp=0xc001f09848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef860, {0x5b54e00, 0xc001174680}, 0xc001f18240, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001f09f28 sp=0xc001f09db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001f09fe0 sp=0xc001f09f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f09fe8 sp=0xc001f09fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 371 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 629 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000184ee8 sp=0xc000184ec8 pc=0x4777ae +runtime.chanrecv(0xc000f26540, 0xc000184fa0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc000184f60 sp=0xc000184ee8 pc=0x441325 +runtime.chanrecv2(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc000184f88 sp=0xc000184f60 pc=0x440f92 +google.golang.org/grpc.(*Server).serverWorker(0xc001ebf4a0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:593 +0x85 fp=0xc000184fc0 sp=0xc000184f88 pc=0x11744a5 +google.golang.org/grpc.(*Server).initServerWorkers.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0x34 fp=0xc000184fe0 sp=0xc000184fc0 pc=0x1174994 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000184fe8 sp=0xc000184fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).initServerWorkers in goroutine 559 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0xa5 + +goroutine 565 [select]: +runtime.gopark(0xc001ddd990?, 0x2?, 0x18?, 0xd8?, 0xc001ddd944?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ddd7c0 sp=0xc001ddd7a0 pc=0x4777ae +runtime.selectgo(0xc001ddd990, 0xc001ddd940, 0x4b0b59?, 0x0, 0x100000001?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ddd910 sp=0xc001ddd7c0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0019cbd40) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001ddd9c0 sp=0xc001ddd910 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bf5ee0, {0x432c3a0?, 0xc0017e0600}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc001dddc20 sp=0xc001ddd9c0 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0019cbbb4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001dddc80 sp=0xc001dddc20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc0019cbb00, 0xc001ddde38, 0xc001ddde08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc001dddd28 sp=0xc001dddc80 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019cbb00, {0x432c3a0, 0xc0017e0600}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001ddde68 sp=0xc001dddd28 pc=0x119077e +github.com/rancher/opni/pkg/apis/management/v1.(*managementWatchClustersClient).Recv(0xc001bf11a0) + /home/ec2-user/opni/pkg/apis/management/v1/management_grpc.pb.go:184 +0x66 fp=0xc001dddea8 sp=0xc001ddde68 pc=0x177c726 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).watchGlobalCluster(0xc0005a8c80, {0x5b6f168, 0xc000fe1300}, 0xc000b529b0?) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/management.go:124 +0x1de fp=0xc001dddf70 sp=0xc001dddea8 pc=0x396477e +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).useWatchers.func1() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:246 +0x55 fp=0xc001dddfc0 sp=0xc001dddf70 pc=0x3973035 +github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents.func1() + /home/ec2-user/opni/pkg/management/watcher.go:27 +0x2f fp=0xc001dddfe0 sp=0xc001dddfc0 pc=0x23220cf +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001dddfe8 sp=0xc001dddfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents in goroutine 177 + /home/ec2-user/opni/pkg/management/watcher.go:26 +0x54 + +goroutine 630 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010e99e0 sp=0xc0010e99c0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0010e9a18 sp=0xc0010e99e0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a588, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0010e9a38 sp=0xc0010e9a18 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001651920, 0x592701?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0010e9a80 sp=0xc0010e9a38 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc001651900) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0010e9b70 sp=0xc0010e9a80 pc=0x513f65 +net.(*netFD).accept(0xc001651900) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0010e9c68 sp=0xc0010e9b70 pc=0x8967be +net.(*TCPListener).accept(0xc001fec100) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc0010e9cb0 sp=0xc0010e9c68 pc=0x8c383e +net.(*TCPListener).Accept(0xc001fec100) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc0010e9d20 sp=0xc0010e9cb0 pc=0x8c1ba5 +google.golang.org/grpc.(*Server).Serve(0xc001ebf4a0, {0x5b2a6d0, 0xc001fec100}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc0010e9f50 sp=0xc0010e9d20 pc=0x117789e +github.com/rancher/opni/pkg/gateway.(*GatewayGRPCServer).ListenAndServe.func1() + /home/ec2-user/opni/pkg/gateway/grpc.go:105 +0x47 fp=0xc0010e9fa0 sp=0xc0010e9f50 pc=0x22b4c27 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0010e9fe0 sp=0xc0010e9fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010e9fe8 sp=0xc0010e9fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 559 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 149 [select]: +runtime.gopark(0xc000f69da8?, 0x2?, 0x60?, 0x98?, 0xc000f69a94?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f69818 sp=0xc000f697f8 pc=0x4777ae +runtime.selectgo(0xc000f69da8, 0xc000f69a90, 0x0?, 0x0, 0x1860a0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000f69968 sp=0xc000f69818 pc=0x48916b +github.com/hashicorp/go-plugin.Serve(0xc000bc3e00) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:474 +0x2091 fp=0xc000f69fc0 sp=0xc000f69968 pc=0x21665b1 +github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins.func4() + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x34 fp=0xc000f69fe0 sp=0xc000f69fc0 pc=0x24cd234 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f69fe8 sp=0xc000f69fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins in goroutine 65 + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x76b + +goroutine 150 [select]: +runtime.gopark(0xc001083f48?, 0x2?, 0x20?, 0x3f?, 0xc001083f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001083d80 sp=0xc001083d60 pc=0x4777ae +runtime.selectgo(0xc001083f48, 0xc001083f00, 0x0?, 0x0, 0x20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001083ed0 sp=0xc001083d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).Recv(0xc000c4a630) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:124 +0xeb fp=0xc001083f78 sp=0xc001083ed0 pc=0x2154feb +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc000451540) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc001083fc0 sp=0xc001083f78 pc=0x215759f +github.com/hashicorp/go-plugin.(*GRPCServer).Init.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x34 fp=0xc001083fe0 sp=0xc001083fc0 pc=0x215adf4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001083fe8 sp=0xc001083fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*GRPCServer).Init in goroutine 149 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x89d + +goroutine 479 [select]: +runtime.gopark(0xc001587e70?, 0x2?, 0x0?, 0x0?, 0xc001587df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002239c58 sp=0xc002239c38 pc=0x4777ae +runtime.selectgo(0xc002239e70, 0xc001587df0, 0xc001587e00?, 0x0, 0xc0010b8000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002239da8 sp=0xc002239c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00139d4f0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002239ea0 sp=0xc002239da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000611880) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002239f58 sp=0xc002239ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc002239fe0 sp=0xc002239f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002239fe8 sp=0xc002239fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 392 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 689 [chan receive]: +runtime.gopark(0x110a2c12646e656b?, 0x43646e656b636162?, 0x6f?, 0x6d?, 0x5209280320011873?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017d3d60 sp=0xc0017d3d40 pc=0x4777ae +runtime.chanrecv(0xc0000672c0, 0xc0017d3e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0017d3dd8 sp=0xc0017d3d60 pc=0x441325 +runtime.chanrecv1(0x696e6f4d6e6f6974?, 0x6142676e69726f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0017d3e00 sp=0xc0017d3dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc0017d3fe0 sp=0xc0017d3e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017d3fe8 sp=0xc0017d3fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 153 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a6f940 sp=0xc000a6f920 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000a6f978 sp=0xc000a6f940 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6ede8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000a6f998 sp=0xc000a6f978 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000c42920, 0xc000a6fa01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000a6f9e0 sp=0xc000a6f998 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc000c42900) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc000a6fad0 sp=0xc000a6f9e0 pc=0x513f65 +net.(*netFD).accept(0xc000c42900) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc000a6fbc8 sp=0xc000a6fad0 pc=0x8967be +net.(*UnixListener).accept(0xc000c4a270) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc000a6fc08 sp=0xc000a6fbc8 pc=0x8cedfe +net.(*UnixListener).Accept(0xc000c4a270) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc000a6fc78 sp=0xc000a6fc08 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc000b74b00) + :1 +0x44 fp=0xc000a6fcb0 sp=0xc000a6fc78 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc000c401e0, {0x5b2c6a0, 0xc000b74b00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc000a6fee0 sp=0xc000a6fcb0 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCServer).Serve(0xc000bbcc60, {0x5b2c6a0, 0xc000b74b00}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:153 +0xdd fp=0xc000a6ff98 sp=0xc000a6fee0 pc=0x215b33d +github.com/hashicorp/go-plugin.Serve.func6() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x62 fp=0xc000a6ffe0 sp=0xc000a6ff98 pc=0x2166b62 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a6ffe8 sp=0xc000a6ffe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.Serve in goroutine 149 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x1f67 + +goroutine 1127 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011a28d8 sp=0xc0011a28b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0011a2910 sp=0xc0011a28d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163bc30, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0011a2930 sp=0xc0011a2910 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00193b220, 0xc001838001?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0011a2978 sp=0xc0011a2930 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc00193b200) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0011a2a68 sp=0xc0011a2978 pc=0x513f65 +net.(*netFD).accept(0xc00193b200) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0011a2b60 sp=0xc0011a2a68 pc=0x8967be +net.(*TCPListener).accept(0xc000b30e00) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc0011a2ba8 sp=0xc0011a2b60 pc=0x8c383e +net.(*TCPListener).Accept(0xc000b30e00) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc0011a2c18 sp=0xc0011a2ba8 pc=0x8c1ba5 +crypto/tls.(*listener).Accept(0xc000126f90) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/tls.go:66 +0x43 fp=0xc0011a2c70 sp=0xc0011a2c18 pc=0x9b2d43 +net/http.(*onceCloseListener).Accept(0xc00294ac60) + :1 +0x44 fp=0xc0011a2ca8 sp=0xc0011a2c70 pc=0xab5544 +net/http.(*Server).Serve(0xc001344e10, {0x5b2a5b0, 0xc000126f90}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc0011a2ea0 sp=0xc0011a2ca8 pc=0xa7859f +net/http.Serve(...) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:2595 +github.com/gin-gonic/gin.(*Engine).RunListener(0xc00190eea0, {0x5b2a5b0, 0xc000126f90}) + /home/ec2-user/go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/gin.go:565 +0x370 fp=0xc0011a2f90 sp=0xc0011a2ea0 pc=0x171e870 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure.func1() + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:72 +0x5c fp=0xc0011a2fe0 sp=0xc0011a2f90 pc=0x262a57c +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011a2fe8 sp=0xc0011a2fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure in goroutine 848 + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:71 +0x26d + +goroutine 542 [select]: +runtime.gopark(0xc001ba3f70?, 0x2?, 0xb4?, 0x1b?, 0xc001ba3ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ba3d20 sp=0xc001ba3d00 pc=0x4777ae +runtime.selectgo(0xc001ba3f70, 0xc001ba3eb8, 0xc0009d3fe0?, 0x0, 0xc0005a8e30?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ba3e70 sp=0xc001ba3d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000ee7400, {0x5b426e0, 0xc00010f680}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001ba3fa0 sp=0xc001ba3e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001ba3fe0 sp=0xc001ba3fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ba3fe8 sp=0xc001ba3fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 434 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 527 [select]: +runtime.gopark(0xc000a51f70?, 0x4?, 0x54?, 0xb6?, 0xc000a51dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a51c00 sp=0xc000a51be0 pc=0x4777ae +runtime.selectgo(0xc000a51f70, 0xc000a51dd0, 0x3920159?, 0x0, 0xc000931c00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a51d50 sp=0xc000a51c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0019e4820) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc000a51fc0 sp=0xc000a51d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc000a51fe0 sp=0xc000a51fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a51fe8 sp=0xc000a51fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 501 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 458 [select]: +runtime.gopark(0xc001169448?, 0x2?, 0xd0?, 0x92?, 0xc0011693fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001169278 sp=0xc001169258 pc=0x4777ae +runtime.selectgo(0xc001169448, 0xc0011693f8, 0xc000813988?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011693c8 sp=0xc001169278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0010dcc60) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001169478 sp=0xc0011693c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001676750, {0x42219c0?, 0xc000ff8b40}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0011696d8 sp=0xc001169478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0010dc894?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001169738 sp=0xc0011696d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc0010dc7e0, 0xc0011698f0, 0xc0011698c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc0011697e0 sp=0xc001169738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010dc7e0, {0x42219c0, 0xc000ff8b40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001169920 sp=0xc0011697e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc000ff8cc0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc000ff8a50}, {0x42219c0, 0xc000ff8b40}, 0x0?, {0xc000996a60, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc0011699c0 sp=0xc001169920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc000762460}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc000ff8a50}, {0x42219c0, 0xc000ff8b40}, 0xc000f7a000, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc001169d30 sp=0xc0011699c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc000f7a000, {0x5b426e0, 0xc000762460}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc000ff8a50}, {0x42219c0, 0xc000ff8b40}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc001169e58 sp=0xc001169d30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseKeyValueStore(0xc00158aa70, {0x5b426e0, 0xc000762460}, 0x0?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:69 +0xee fp=0xc001169ee8 sp=0xc001169e58 pc=0x21ab96e +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeKeyValueStore.func2(0x3) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:219 +0xf2 fp=0xc001169f58 sp=0xc001169ee8 pc=0x21a85d2 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc001169fe0 sp=0xc001169f58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001169fe8 sp=0xc001169fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 456 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 1191 [select]: +runtime.gopark(0xc00189be70?, 0x2?, 0x0?, 0x0?, 0xc00189bdf4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00189bc58 sp=0xc00189bc38 pc=0x4777ae +runtime.selectgo(0xc00189be70, 0xc00189bdf0, 0xc00189be00?, 0x0, 0xc001ce3b58?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00189bda8 sp=0xc00189bc58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0015fbea0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00189bea0 sp=0xc00189bda8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0014fae70) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00189bf58 sp=0xc00189bea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc00189bfe0 sp=0xc00189bf58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00189bfe8 sp=0xc00189bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1073 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 97 [select]: +runtime.gopark(0xc000f6ff70?, 0x2?, 0xb4?, 0x1b?, 0xc000f6febc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f6fd20 sp=0xc000f6fd00 pc=0x4777ae +runtime.selectgo(0xc000f6ff70, 0xc000f6feb8, 0x0?, 0x0, 0x472ddb?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000f6fe70 sp=0xc000f6fd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000996be0, {0x5b426e0, 0xc000762500}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc000f6ffa0 sp=0xc000f6fe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc000f6ffe0 sp=0xc000f6ffa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f6ffe8 sp=0xc000f6ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 162 [select]: +runtime.gopark(0xc000d13f70?, 0x2?, 0xb4?, 0x1b?, 0xc000d13ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010ebd20 sp=0xc0010ebd00 pc=0x4777ae +runtime.selectgo(0xc0010ebf70, 0xc000d13eb8, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010ebe70 sp=0xc0010ebd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000996c40, {0x5b426e0, 0xc000762550}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0010ebfa0 sp=0xc0010ebe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0010ebfe0 sp=0xc0010ebfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010ebfe8 sp=0xc0010ebfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 159 [select]: +runtime.gopark(0xc000f6bda8?, 0x2?, 0x60?, 0xb8?, 0xc000f6ba94?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f6b818 sp=0xc000f6b7f8 pc=0x4777ae +runtime.selectgo(0xc000f6bda8, 0xc000f6ba90, 0x0?, 0x0, 0x1860a0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000f6b968 sp=0xc000f6b818 pc=0x48916b +github.com/hashicorp/go-plugin.Serve(0xc000c4efc0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:474 +0x2091 fp=0xc000f6bfc0 sp=0xc000f6b968 pc=0x21665b1 +github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins.func4() + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x34 fp=0xc000f6bfe0 sp=0xc000f6bfc0 pc=0x24cd234 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f6bfe8 sp=0xc000f6bfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins in goroutine 65 + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x76b + +goroutine 168 [select]: +runtime.gopark(0xc000a4bfb0?, 0x2?, 0xb4?, 0x1b?, 0xc000a4bf74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a4bde8 sp=0xc000a4bdc8 pc=0x4777ae +runtime.selectgo(0xc000a4bfb0, 0xc000a4bf70, 0xc000a4bfd0?, 0x0, 0xc0004bbd50?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a4bf38 sp=0xc000a4bde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc000a4bfe0 sp=0xc000a4bf38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a4bfe8 sp=0xc000a4bfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 160 [select]: +runtime.gopark(0xc000d08f48?, 0x2?, 0x20?, 0x8f?, 0xc000d08f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d08d80 sp=0xc000d08d60 pc=0x4777ae +runtime.selectgo(0xc000d08f48, 0xc000d08f00, 0x0?, 0x0, 0x20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d08ed0 sp=0xc000d08d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).Recv(0xc000c70510) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:124 +0xeb fp=0xc000d08f78 sp=0xc000d08ed0 pc=0x2154feb +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc000b763c0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc000d08fc0 sp=0xc000d08f78 pc=0x215759f +github.com/hashicorp/go-plugin.(*GRPCServer).Init.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x34 fp=0xc000d08fe0 sp=0xc000d08fc0 pc=0x215adf4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d08fe8 sp=0xc000d08fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*GRPCServer).Init in goroutine 159 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x89d + +goroutine 487 [IO wait]: +runtime.gopark(0xc00157af88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00157af40 sp=0xc00157af20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00157af78 sp=0xc00157af40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6ea08, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00157af98 sp=0xc00157af78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0014d3820, 0xc0014a0d80?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00157afe0 sp=0xc00157af98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0014d3800, {0xc0014a0d80, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00157b0d0 sp=0xc00157afe0 pc=0x50e5a5 +net.(*netFD).Read(0xc0014d3800, {0xc0014a0d80, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00157b130 sp=0xc00157b0d0 pc=0x89342b +net.(*conn).Read(0xc000640018, {0xc0014a0d80, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00157b1b8 sp=0xc00157b130 pc=0x8b176d +net.(*UnixConn).Read(0xc000640018, {0xc0014a0d80, 0x240, 0x240}) + :1 +0x54 fp=0xc00157b200 sp=0xc00157b1b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0004a7ba8, {0xc0014a0d80, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc00157b260 sp=0xc00157b200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000dc0d28, {0x5b0f6c0, 0xc0004a7ba8}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc00157b2c8 sp=0xc00157b260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000dc0a80, {0x5b0f840?, 0xc000640018}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc00157b338 sp=0xc00157b2c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000dc0a80, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc00157b968 sp=0xc00157b338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000dc0a80, {0xc001776000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc00157ba50 sp=0xc00157b968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0016351e0, {0xc001776000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc00157ba98 sp=0xc00157ba50 pc=0x10969ed +bufio.(*Reader).Read(0xc0011a45a0, {0xc000e4ef20, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00157bb58 sp=0xc00157ba98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0011a45a0}, {0xc000e4ef20, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00157bba0 sp=0xc00157bb58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000e4ef20, 0x9, 0x9}, {0x5b0f0a0, 0xc0011a45a0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00157bc40 sp=0xc00157bba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000e4eee0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00157bd78 sp=0xc00157bc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0012c5a00, 0x1000?, 0x5b0ee40?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc00157bec8 sp=0xc00157bd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00141a3c0, {0x5b54e00?, 0xc0012c5a00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc00157bf80 sp=0xc00157bec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc00157bfe0 sp=0xc00157bf80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00157bfe8 sp=0xc00157bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 453 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 486 [select]: +runtime.gopark(0xc000d3af70?, 0x4?, 0xa9?, 0x5?, 0xc000d3add8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d3ac00 sp=0xc000d3abe0 pc=0x4777ae +runtime.selectgo(0xc000d3af70, 0xc000d3add0, 0x87bab8?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d3ad50 sp=0xc000d3ac00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0012c5a00) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc000d3afc0 sp=0xc000d3ad50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc000d3afe0 sp=0xc000d3afc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d3afe8 sp=0xc000d3afe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 453 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 101 [select]: +runtime.gopark(0xc000d6f748?, 0x2?, 0xd0?, 0xf5?, 0xc000d6f6fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d6f578 sp=0xc000d6f558 pc=0x4777ae +runtime.selectgo(0xc000d6f748, 0xc000d6f6f8, 0xc000934940?, 0x0, 0xc000d16198?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d6f6c8 sp=0xc000d6f578 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000926120) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc000d6f778 sp=0xc000d6f6c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00094c680, {0x43dd860?, 0xc000ee4dc0}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc000d6f9d8 sp=0xc000d6f778 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000c05974?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc000d6fa38 sp=0xc000d6f9d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc000c058c0, 0xc000d6fbf0, 0xc000d6fbc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc000d6fae0 sp=0xc000d6fa38 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c058c0, {0x43dd860, 0xc000ee4dc0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc000d6fc20 sp=0xc000d6fae0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc000fe6ec0, {0x43dd860, 0xc000ee4dc0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc000d6fde8 sp=0xc000d6fc20 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamClient).Recv(0xc000b6f0f0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:132 +0x66 fp=0xc000d6fe28 sp=0xc000d6fde8 pc=0x2102606 +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream(0xc000fe6b80) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:197 +0x2e8 fp=0xc000d6ffc0 sp=0xc000d6fe28 pc=0x2155468 +github.com/hashicorp/go-plugin.newGRPCClient.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x34 fp=0xc000d6ffe0 sp=0xc000d6ffc0 pc=0x2159954 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d6ffe8 sp=0xc000d6ffe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 154 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x62e + +goroutine 179 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000e81940 sp=0xc000e81920 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000e81978 sp=0xc000e81940 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6e720, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000e81998 sp=0xc000e81978 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000c43ea0, 0xc000e81a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000e819e0 sp=0xc000e81998 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc000c43e80) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc000e81ad0 sp=0xc000e819e0 pc=0x513f65 +net.(*netFD).accept(0xc000c43e80) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc000e81bc8 sp=0xc000e81ad0 pc=0x8967be +net.(*UnixListener).accept(0xc000c70150) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc000e81c08 sp=0xc000e81bc8 pc=0x8cedfe +net.(*UnixListener).Accept(0xc000c70150) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc000e81c78 sp=0xc000e81c08 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc000b75280) + :1 +0x44 fp=0xc000e81cb0 sp=0xc000e81c78 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc000c40d20, {0x5b2c6a0, 0xc000b75280}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc000e81ee0 sp=0xc000e81cb0 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCServer).Serve(0xc000bbcf30, {0x5b2c6a0, 0xc000b75280}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:153 +0xdd fp=0xc000e81f98 sp=0xc000e81ee0 pc=0x215b33d +github.com/hashicorp/go-plugin.Serve.func6() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x62 fp=0xc000e81fe0 sp=0xc000e81f98 pc=0x2166b62 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000e81fe8 sp=0xc000e81fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.Serve in goroutine 159 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x1f67 + +goroutine 169 [select]: +runtime.gopark(0xc000d04fb0?, 0x2?, 0xb0?, 0x4d?, 0xc000d04ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d04d58 sp=0xc000d04d38 pc=0x4777ae +runtime.selectgo(0xc000d04fb0, 0xc000d04ee0, 0xc000d04f98?, 0x0, 0xc000dddfb0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d04ea8 sp=0xc000d04d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc000d04fe0 sp=0xc000d04ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d04fe8 sp=0xc000d04fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 166 [IO wait]: +runtime.gopark(0xc000e90fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000e90f98 sp=0xc000e90f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000e90fd0 sp=0xc000e90f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6e248, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000e90ff0 sp=0xc000e90fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d72620, 0xc001749800?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000e91038 sp=0xc000e90ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000d72600, {0xc001749800, 0x4800, 0x4800}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc000e91128 sp=0xc000e91038 pc=0x50e5a5 +net.(*netFD).Read(0xc000d72600, {0xc001749800, 0x4800, 0x4800}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc000e91188 sp=0xc000e91128 pc=0x89342b +net.(*conn).Read(0xc00007d870, {0xc001749800, 0x4800, 0x4800}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc000e91210 sp=0xc000e91188 pc=0x8b176d +net.(*UnixConn).Read(0xc00007d870, {0xc001749800, 0x4800, 0x4800}) + :1 +0x54 fp=0xc000e91258 sp=0xc000e91210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0008630f8, {0xc001749800, 0x4800, 0x4800}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc000e912b8 sp=0xc000e91258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000c7e9a8, {0x5b0f6c0, 0xc0008630f8}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc000e91320 sp=0xc000e912b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000c7e700, {0x5b0f840?, 0xc00007d870}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc000e91390 sp=0xc000e91320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000c7e700, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc000e919c0 sp=0xc000e91390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000c7e700, {0xc000fb8000, 0x8000, 0xc000bdc050?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc000e91aa8 sp=0xc000e919c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0009970e0, {0xc000fb8000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc000e91af0 sp=0xc000e91aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc000f1b560, {0xc000db2820, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc000e91bb0 sp=0xc000e91af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000f1b560}, {0xc000db2820, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000e91bf8 sp=0xc000e91bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000db2820, 0x9, 0x9}, {0x5b0f0a0, 0xc000f1b560}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc000e91c98 sp=0xc000e91bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000db27e0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc000e91dd0 sp=0xc000e91c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000eecfc0, 0xc000c7e700?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc000e91fb0 sp=0xc000e91dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc000e91fe0 sp=0xc000e91fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000e91fe8 sp=0xc000e91fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 163 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 185 [select]: +runtime.gopark(0xc002a7fe48?, 0x2?, 0x0?, 0x0?, 0xc002a7fdcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002a7fc30 sp=0xc002a7fc10 pc=0x4777ae +runtime.selectgo(0xc002a7fe48, 0xc002a7fdc8, 0xc002a7fdd8?, 0x0, 0xc000dc0000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002a7fd80 sp=0xc002a7fc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000bdda90, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002a7fe78 sp=0xc002a7fd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004aad20) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002a7ff30 sp=0xc002a7fe78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc002a7ffe0 sp=0xc002a7ff30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002a7ffe8 sp=0xc002a7ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 181 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 135 [chan receive]: +runtime.gopark(0x0?, 0x10?, 0xa0?, 0xf0?, 0xc002a4b5d4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d3bd10 sp=0xc000d3bcf0 pc=0x4777ae +runtime.chanrecv(0xc000c4f320, 0xc000d3be40, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc000d3bd88 sp=0xc000d3bd10 pc=0x441325 +runtime.chanrecv2(0x3b9aca00?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc000d3bdb0 sp=0xc000d3bd88 pc=0x440f92 +github.com/hashicorp/go-plugin.pidWait(0x1860a0?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/process.go:20 +0xdb fp=0xc000d3be68 sp=0xc000d3bdb0 pc=0x216033b +github.com/hashicorp/go-plugin.(*Client).reattach.func1(0x215adf4?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:853 +0xd8 fp=0xc000d3bfb8 sp=0xc000d3be68 pc=0x21528b8 +github.com/hashicorp/go-plugin.(*Client).reattach.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:862 +0x42 fp=0xc000d3bfe0 sp=0xc000d3bfb8 pc=0x21527a2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d3bfe8 sp=0xc000d3bfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*Client).reattach in goroutine 134 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:846 +0x365 + +goroutine 142 [select]: +runtime.gopark(0xc000d36fb0?, 0x2?, 0xb0?, 0x6d?, 0xc000d36ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d36d58 sp=0xc000d36d38 pc=0x4777ae +runtime.selectgo(0xc000d36fb0, 0xc000d36ee0, 0xc000d36f98?, 0x0, 0xc000d183c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d36ea8 sp=0xc000d36d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc000d36fe0 sp=0xc000d36ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d36fe8 sp=0xc000d36fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 136 [select]: +runtime.gopark(0xc000e95f70?, 0x2?, 0xb4?, 0x1b?, 0xc000e95ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000e95d20 sp=0xc000e95d00 pc=0x4777ae +runtime.selectgo(0xc000e95f70, 0xc000e95eb8, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000e95e70 sp=0xc000e95d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0005fad80, {0x5b426e0, 0xc000bdcb40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc000e95fa0 sp=0xc000e95e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc000e95fe0 sp=0xc000e95fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000e95fe8 sp=0xc000e95fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 137 [select]: +runtime.gopark(0xc000a71f70?, 0x2?, 0xb4?, 0x1b?, 0xc000a71ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a71d20 sp=0xc000a71d00 pc=0x4777ae +runtime.selectgo(0xc000a71f70, 0xc000a71eb8, 0x0?, 0x0, 0x440f72?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a71e70 sp=0xc000a71d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0005fade0, {0x5b426e0, 0xc000bdcb90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc000a71fa0 sp=0xc000a71e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc000a71fe0 sp=0xc000a71fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a71fe8 sp=0xc000a71fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 141 [select]: +runtime.gopark(0xc000d37fb0?, 0x2?, 0xb4?, 0x1b?, 0xc000d37f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d37de8 sp=0xc000d37dc8 pc=0x4777ae +runtime.selectgo(0xc000d37fb0, 0xc000d37f70, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d37f38 sp=0xc000d37de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc000d37fe0 sp=0xc000d37f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d37fe8 sp=0xc000d37fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 139 [select]: +runtime.gopark(0xc000d09f48?, 0x2?, 0x0?, 0xda?, 0xc000d09f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d09d80 sp=0xc000d09d60 pc=0x4777ae +runtime.selectgo(0xc000d09f48, 0xc000d09f00, 0xc000d09f08?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d09ed0 sp=0xc000d09d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).Recv(0xc0008c4440) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:234 +0xeb fp=0xc000d09f78 sp=0xc000d09ed0 pc=0x2155d2b +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc000bdd540) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc000d09fc0 sp=0xc000d09f78 pc=0x215759f +github.com/hashicorp/go-plugin.newGRPCClient.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x34 fp=0xc000d09fe0 sp=0xc000d09fc0 pc=0x21599b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d09fe8 sp=0xc000d09fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 134 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x5b7 + +goroutine 140 [select]: +runtime.gopark(0xc000d6b748?, 0x2?, 0xd0?, 0xb5?, 0xc000d6b6fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d6b578 sp=0xc000d6b558 pc=0x4777ae +runtime.selectgo(0xc000d6b748, 0xc000d6b6f8, 0xc000a52580?, 0x0, 0xc000ed2e18?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d6b6c8 sp=0xc000d6b578 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000926360) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc000d6b778 sp=0xc000d6b6c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00089fd40, {0x43dd860?, 0xc000ee4f00}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc000d6b9d8 sp=0xc000d6b778 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00094e894?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc000d6ba38 sp=0xc000d6b9d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00094e7e0, 0xc000d6bbf0, 0xc000d6bbc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc000d6bae0 sp=0xc000d6ba38 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00094e7e0, {0x43dd860, 0xc000ee4f00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc000d6bc20 sp=0xc000d6bae0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc000fe6fc0, {0x43dd860, 0xc000ee4f00}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc000d6bde8 sp=0xc000d6bc20 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamClient).Recv(0xc000b6f1c0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:132 +0x66 fp=0xc000d6be28 sp=0xc000d6bde8 pc=0x2102606 +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream(0xc0008c4440) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:197 +0x2e8 fp=0xc000d6bfc0 sp=0xc000d6be28 pc=0x2155468 +github.com/hashicorp/go-plugin.newGRPCClient.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x34 fp=0xc000d6bfe0 sp=0xc000d6bfc0 pc=0x2159954 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d6bfe8 sp=0xc000d6bfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 134 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x62e + +goroutine 105 [IO wait]: +runtime.gopark(0xc000e92fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000e92f98 sp=0xc000e92f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000e92fd0 sp=0xc000e92f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6e058, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000e92ff0 sp=0xc000e92fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d163a0, 0xc001a0e000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000e93038 sp=0xc000e92ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000d16380, {0xc001a0e000, 0xa000, 0xa000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc000e93128 sp=0xc000e93038 pc=0x50e5a5 +net.(*netFD).Read(0xc000d16380, {0xc001a0e000, 0xa000, 0xa000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc000e93188 sp=0xc000e93128 pc=0x89342b +net.(*conn).Read(0xc000991078, {0xc001a0e000, 0xa000, 0xa000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc000e93210 sp=0xc000e93188 pc=0x8b176d +net.(*UnixConn).Read(0xc000991078, {0xc001a0e000, 0xa000, 0xa000}) + :1 +0x54 fp=0xc000e93258 sp=0xc000e93210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc00290e0d8, {0xc001a0e000, 0xa000, 0xa000}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc000e932b8 sp=0xc000e93258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000f889a8, {0x5b0f6c0, 0xc00290e0d8}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc000e93320 sp=0xc000e932b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000f88700, {0x5b0f840?, 0xc000991078}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc000e93390 sp=0xc000e93320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000f88700, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc000e939c0 sp=0xc000e93390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000f88700, {0xc000fd8000, 0x8000, 0xc00296def0?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc000e93aa8 sp=0xc000e939c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc000ee6820, {0xc000fd8000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc000e93af0 sp=0xc000e93aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc000d65a40, {0xc000f8e040, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc000e93bb0 sp=0xc000e93af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000d65a40}, {0xc000f8e040, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000e93bf8 sp=0xc000e93bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000f8e040, 0x9, 0x9}, {0x5b0f0a0, 0xc000d65a40}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc000e93c98 sp=0xc000e93bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000f8e000) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc000e93dd0 sp=0xc000e93c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000b50fc0, 0xc000f88700?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc000e93fb0 sp=0xc000e93dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc000e93fe0 sp=0xc000e93fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000e93fe8 sp=0xc000e93fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 138 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 188 [select]: +runtime.gopark(0xc000e7d310?, 0x3?, 0xd9?, 0xea?, 0xc000e7d292?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000e7d0f0 sp=0xc000e7d0d0 pc=0x4777ae +runtime.selectgo(0xc000e7d310, 0xc000e7d28c, 0x46216a2?, 0x0, 0x4?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000e7d240 sp=0xc000e7d0f0 pc=0x48916b +github.com/hashicorp/go-plugin.(*grpcStdioServer).StreamStdio(0xc000bfd4e0, 0x42219c0?, {0x5b4ebb0, 0xc000b6f570}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:61 +0x1e5 fp=0xc000e7d350 sp=0xc000e7d240 pc=0x215bbc5 +github.com/hashicorp/go-plugin/internal/plugin._GRPCStdio_StreamStdio_Handler({0x409e580?, 0xc000bfd4e0}, {0x5b4a6f0, 0xc000ee69c0}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:185 +0xf7 fp=0xc000e7d3b0 sp=0xc000e7d350 pc=0x2104077 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x409e580, 0xc000bfd4e0}, {0x5b4ab30, 0xc000d19170}, 0xc000f1ec78?, 0x47efca8) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc000e7d400 sp=0xc000e7d3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x409e580, 0xc000bfd4e0}, {0x5b4ab30, 0xc000d19170}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc000e7d490 sp=0xc000e7d400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x409e580, 0xc000bfd4e0}, {0x5b492f8, 0xc0009ec780}, 0xc000f1ec78, 0xc000fe7040) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc000e7d7d0 sp=0xc000e7d490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x409e580, 0xc000bfd4e0}, {0x5b492f8, 0xc0009ec780}, 0x3f252a0?, 0xc000b6f1d0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc000e7d848 sp=0xc000e7d7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc000e51320, 0xc000c4a870, 0x770cba0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc000e7ddb0 sp=0xc000e7d848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc000e51320, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc000e7df28 sp=0xc000e7ddb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc000e7dfe0 sp=0xc000e7df28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000e7dfe8 sp=0xc000e7dfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 167 [select]: +runtime.gopark(0xc000c59e70?, 0x2?, 0x0?, 0x0?, 0xc000c59df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020dfc58 sp=0xc0020dfc38 pc=0x4777ae +runtime.selectgo(0xc0020dfe70, 0xc000c59df0, 0xc000c59e00?, 0x0, 0xc000c7e700?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020dfda8 sp=0xc0020dfc58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000763360, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0020dfea0 sp=0xc0020dfda8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004bbd50) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0020dff58 sp=0xc0020dfea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc0020dffe0 sp=0xc0020dff58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020dffe8 sp=0xc0020dffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 163 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 182 [select]: +runtime.gopark(0xc001f3fe48?, 0x2?, 0x0?, 0x0?, 0xc001f3fdcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e69c30 sp=0xc001e69c10 pc=0x4777ae +runtime.selectgo(0xc001e69e48, 0xc001f3fdc8, 0xc001f3fdd8?, 0x0, 0xc000f88000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e69d80 sp=0xc001e69c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000b76550, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001e69e78 sp=0xc001e69d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004aac40) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001e69f30 sp=0xc001e69e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc001e69fe0 sp=0xc001e69f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e69fe8 sp=0xc001e69fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 180 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 183 [select]: +runtime.gopark(0xc000d3cf70?, 0x4?, 0x10?, 0x0?, 0xc000d3cdd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d3cc00 sp=0xc000d3cbe0 pc=0x4777ae +runtime.selectgo(0xc000d3cf70, 0xc000d3cdd0, 0xc000c48fe0?, 0x0, 0x112e2bb?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d3cd50 sp=0xc000d3cc00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000c7cb60) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc000d3cfc0 sp=0xc000d3cd50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc000d3cfe0 sp=0xc000d3cfc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d3cfe8 sp=0xc000d3cfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 180 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 184 [IO wait]: +runtime.gopark(0xc000e82f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000e82f40 sp=0xc000e82f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000e82f78 sp=0xc000e82f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6e150, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000e82f98 sp=0xc000e82f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000dba0a0, 0xc00024c800?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000e82fe0 sp=0xc000e82f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000dba080, {0xc00024c800, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc000e830d0 sp=0xc000e82fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc000dba080, {0xc00024c800, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc000e83130 sp=0xc000e830d0 pc=0x89342b +net.(*conn).Read(0xc000b799b8, {0xc00024c800, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc000e831b8 sp=0xc000e83130 pc=0x8b176d +net.(*UnixConn).Read(0xc000b799b8, {0xc00024c800, 0x800, 0x800}) + :1 +0x54 fp=0xc000e83200 sp=0xc000e831b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc000013f80, {0xc00024c800, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc000e83260 sp=0xc000e83200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000f882a8, {0x5b0f6c0, 0xc000013f80}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc000e832c8 sp=0xc000e83260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000f88000, {0x5b0f840?, 0xc000b799b8}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc000e83338 sp=0xc000e832c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000f88000, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc000e83968 sp=0xc000e83338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000f88000, {0xc000fd0000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc000e83a50 sp=0xc000e83968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc000b756c0, {0xc000fd0000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc000e83a98 sp=0xc000e83a50 pc=0x10969ed +bufio.(*Reader).Read(0xc000c4f680, {0xc000c49000, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc000e83b58 sp=0xc000e83a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000c4f680}, {0xc000c49000, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000e83ba0 sp=0xc000e83b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000c49000, 0x9, 0x9}, {0x5b0f0a0, 0xc000c4f680}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc000e83c40 sp=0xc000e83ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000c48fc0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc000e83d78 sp=0xc000e83c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000c7cb60, 0x0?, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc000e83ec8 sp=0xc000e83d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc000c401e0, {0x5b54e00?, 0xc000c7cb60}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc000e83f80 sp=0xc000e83ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc000e83fe0 sp=0xc000e83f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000e83fe8 sp=0xc000e83fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 180 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 106 [select]: +runtime.gopark(0xc00111fe70?, 0x2?, 0x0?, 0x0?, 0xc00111fdf4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00111fc58 sp=0xc00111fc38 pc=0x4777ae +runtime.selectgo(0xc00111fe70, 0xc00111fdf0, 0xc00111fe00?, 0x0, 0xc000f88700?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00111fda8 sp=0xc00111fc58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000ee4c30, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00111fea0 sp=0xc00111fda8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004b4af0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00111ff58 sp=0xc00111fea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc00111ffe0 sp=0xc00111ff58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00111ffe8 sp=0xc00111ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 138 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 170 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc000ef8df8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000ef8d60 sp=0xc000ef8d40 pc=0x4777ae +runtime.chanrecv(0xc000f26060, 0xc000ef8e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc000ef8dd8 sp=0xc000ef8d60 pc=0x441325 +runtime.chanrecv1(0x448845?, 0x40?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc000ef8e00 sp=0xc000ef8dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc000ef8fe0 sp=0xc000ef8e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000ef8fe8 sp=0xc000ef8fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 171 [select]: +runtime.gopark(0xc000e97790?, 0x2?, 0x18?, 0x76?, 0xc000e97744?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000e975c0 sp=0xc000e975a0 pc=0x4777ae +runtime.selectgo(0xc000e97790, 0xc000e97740, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000e97710 sp=0xc000e975c0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000f8aa20) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc000e977c0 sp=0xc000e97710 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00094c4e0, {0x43c4ea0?, 0xc000fe7000}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc000e97a20 sp=0xc000e977c0 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000c05854?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc000e97a80 sp=0xc000e97a20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc000c057a0, 0xc000ef8c38, 0xc000ef8c08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc000e97b28 sp=0xc000e97a80 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c057a0, {0x43c4ea0, 0xc000fe7000}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc000e97c68 sp=0xc000e97b28 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc000dd8e40, {0x43c4ea0, 0xc000fe7000}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc000e97e30 sp=0xc000e97c68 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCStdioStreamStdioClient).Recv(0xc000813b10) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:160 +0x66 fp=0xc000e97e70 sp=0xc000e97e30 pc=0x2103f26 +github.com/hashicorp/go-plugin.(*grpcStdioClient).Run(0xc000997200, {0x5b0ee60, 0x779c980}, {0x5b0eb80, 0xc00007c038}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:135 +0x11c fp=0xc000e97f80 sp=0xc000e97e70 pc=0x215c15c +github.com/hashicorp/go-plugin.newGRPCClient.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x6c fp=0xc000e97fe0 sp=0xc000e97f80 pc=0x21598ec +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000e97fe8 sp=0xc000e97fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 154 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x84f + +goroutine 186 [select]: +runtime.gopark(0xc000d3df70?, 0x4?, 0x58?, 0xda?, 0xc000d3ddd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d3dc00 sp=0xc000d3dbe0 pc=0x4777ae +runtime.selectgo(0xc000d3df70, 0xc000d3ddd0, 0x40a16a0?, 0x0, 0xc000e9c050?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d3dd50 sp=0xc000d3dc00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000e8c680) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc000d3dfc0 sp=0xc000d3dd50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc000d3dfe0 sp=0xc000d3dfc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d3dfe8 sp=0xc000d3dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 181 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 187 [IO wait]: +runtime.gopark(0xc00109ef88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00109ef40 sp=0xc00109ef20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00109ef78 sp=0xc00109ef40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677eb0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00109ef98 sp=0xc00109ef78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000dba220, 0xc001154000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00109efe0 sp=0xc00109ef98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000dba200, {0xc001154000, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00109f0d0 sp=0xc00109efe0 pc=0x50e5a5 +net.(*netFD).Read(0xc000dba200, {0xc001154000, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00109f130 sp=0xc00109f0d0 pc=0x89342b +net.(*conn).Read(0xc000b799c0, {0xc001154000, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00109f1b8 sp=0xc00109f130 pc=0x8b176d +net.(*UnixConn).Read(0xc000b799c0, {0xc001154000, 0x800, 0x800}) + :1 +0x54 fp=0xc00109f200 sp=0xc00109f1b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0027d83c0, {0xc001154000, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc00109f260 sp=0xc00109f200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000dc02a8, {0x5b0f6c0, 0xc0027d83c0}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc00109f2c8 sp=0xc00109f260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000dc0000, {0x5b0f840?, 0xc000b799c0}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc00109f338 sp=0xc00109f2c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000dc0000, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc00109f968 sp=0xc00109f338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000dc0000, {0xc000e9e000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc00109fa50 sp=0xc00109f968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0005fafe0, {0xc000e9e000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc00109fa98 sp=0xc00109fa50 pc=0x10969ed +bufio.(*Reader).Read(0xc000a528a0, {0xc000477380, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00109fb58 sp=0xc00109fa98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000a528a0}, {0xc000477380, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00109fba0 sp=0xc00109fb58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000477380, 0x9, 0x9}, {0x5b0f0a0, 0xc000a528a0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00109fc40 sp=0xc00109fba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000477340) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00109fd78 sp=0xc00109fc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000e8c680, 0x1177efa?, 0x215b33d?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc00109fec8 sp=0xc00109fd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc000c40d20, {0x5b54e00?, 0xc000e8c680}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc00109ff80 sp=0xc00109fec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc00109ffe0 sp=0xc00109ff80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00109ffe8 sp=0xc00109ffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 181 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 143 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x1188140?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d05d60 sp=0xc000d05d40 pc=0x4777ae +runtime.chanrecv(0xc000c9d2c0, 0xc000d05e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc000d05dd8 sp=0xc000d05d60 pc=0x441325 +runtime.chanrecv1(0x472ddb?, 0xc000bdd7c0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc000d05e00 sp=0xc000d05dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc000d05fe0 sp=0xc000d05e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d05fe8 sp=0xc000d05fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 172 [select]: +runtime.gopark(0xc000a4dfb0?, 0x2?, 0xb4?, 0x1b?, 0xc000a4df74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a4dde8 sp=0xc000a4ddc8 pc=0x4777ae +runtime.selectgo(0xc000a4dfb0, 0xc000a4df70, 0x2?, 0x0, 0x5b42750?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a4df38 sp=0xc000a4dde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc000a4dfe0 sp=0xc000a4df38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a4dfe8 sp=0xc000a4dfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 173 [select]: +runtime.gopark(0xc000a4cfb0?, 0x2?, 0xb0?, 0xcd?, 0xc000a4cee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a4cd58 sp=0xc000a4cd38 pc=0x4777ae +runtime.selectgo(0xc000a4cfb0, 0xc000a4cee0, 0x448855?, 0x0, 0x400a060?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a4cea8 sp=0xc000a4cd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc000a4cfe0 sp=0xc000a4cea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a4cfe8 sp=0xc000a4cfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 107 [select]: +runtime.gopark(0xc000d39fb0?, 0x2?, 0xb4?, 0x1b?, 0xc000d39f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d39de8 sp=0xc000d39dc8 pc=0x4777ae +runtime.selectgo(0xc000d39fb0, 0xc000d39f70, 0xc000b85fd0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d39f38 sp=0xc000d39de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc000d39fe0 sp=0xc000d39f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d39fe8 sp=0xc000d39fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 101 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 174 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000ef9d60 sp=0xc000ef9d40 pc=0x4777ae +runtime.chanrecv(0xc000f26300, 0xc000ef9e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc000ef9dd8 sp=0xc000ef9d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc000ef9e00 sp=0xc000ef9dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc000ef9fe0 sp=0xc000ef9e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000ef9fe8 sp=0xc000ef9fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 108 [select]: +runtime.gopark(0xc001092fb0?, 0x2?, 0xb0?, 0x2d?, 0xc001092ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001092d58 sp=0xc001092d38 pc=0x4777ae +runtime.selectgo(0xc001092fb0, 0xc001092ee0, 0x392effa?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001092ea8 sp=0xc001092d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001092fe0 sp=0xc001092ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001092fe8 sp=0xc001092fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 101 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 109 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001093d60 sp=0xc001093d40 pc=0x4777ae +runtime.chanrecv(0xc001089140, 0xc001093e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001093dd8 sp=0xc001093d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001093e00 sp=0xc001093dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001093fe0 sp=0xc001093e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001093fe8 sp=0xc001093fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 101 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 144 [select]: +runtime.gopark(0xc00109b790?, 0x2?, 0x18?, 0xb6?, 0xc00109b744?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00109b5c0 sp=0xc00109b5a0 pc=0x4777ae +runtime.selectgo(0xc00109b790, 0xc00109b740, 0x1?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00109b710 sp=0xc00109b5c0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00094eea0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc00109b7c0 sp=0xc00109b710 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00089fc70, {0x43c4ea0?, 0xc0008c4880}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc00109ba20 sp=0xc00109b7c0 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00094e774?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc00109ba80 sp=0xc00109ba20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00094e6c0, 0xc000efac38, 0xc000efac08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc00109bb28 sp=0xc00109ba80 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00094e6c0, {0x43c4ea0, 0xc0008c4880}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc00109bc68 sp=0xc00109bb28 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc0008c4600, {0x43c4ea0, 0xc0008c4880}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc00109be30 sp=0xc00109bc68 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCStdioStreamStdioClient).Recv(0xc000e9c1c0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:160 +0x66 fp=0xc00109be70 sp=0xc00109be30 pc=0x2103f26 +github.com/hashicorp/go-plugin.(*grpcStdioClient).Run(0xc0005fb100, {0x5b0ee60, 0x779c980}, {0x5b0eb80, 0xc00007c038}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:135 +0x11c fp=0xc00109bf80 sp=0xc00109be70 pc=0x215c15c +github.com/hashicorp/go-plugin.newGRPCClient.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x6c fp=0xc00109bfe0 sp=0xc00109bf80 pc=0x21598ec +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00109bfe8 sp=0xc00109bfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 134 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x84f + +goroutine 110 [select]: +runtime.gopark(0xc0015f3fa0?, 0x3?, 0xf0?, 0x3d?, 0xc0015f3f3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015f3da8 sp=0xc0015f3d88 pc=0x4777ae +runtime.selectgo(0xc0015f3fa0, 0xc0015f3f34, 0xc000d38f40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0015f3ef8 sp=0xc0015f3da8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:184 +0x1b3 fp=0xc0015f3fe0 sp=0xc0015f3ef8 pc=0x21557f3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015f3fe8 sp=0xc0015f3fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream in goroutine 101 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:182 +0x24a + +goroutine 145 [select]: +runtime.gopark(0xc000a4afb0?, 0x2?, 0xb4?, 0x1b?, 0xc000a4af74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a4ade8 sp=0xc000a4adc8 pc=0x4777ae +runtime.selectgo(0xc000a4afb0, 0xc000a4af70, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a4af38 sp=0xc000a4ade8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc000a4afe0 sp=0xc000a4af38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a4afe8 sp=0xc000a4afe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 194 [select]: +runtime.gopark(0xc000efafb0?, 0x2?, 0xb0?, 0xad?, 0xc000efaee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000efad58 sp=0xc000efad38 pc=0x4777ae +runtime.selectgo(0xc000efafb0, 0xc000efaee0, 0x4aead9?, 0x0, 0xc000f1df30?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000efaea8 sp=0xc000efad58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc000efafe0 sp=0xc000efaea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000efafe8 sp=0xc000efafe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 195 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000efbd60 sp=0xc000efbd40 pc=0x4777ae +runtime.chanrecv(0xc000c9d560, 0xc000efbe60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc000efbdd8 sp=0xc000efbd60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc000efbe00 sp=0xc000efbdd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc000efbfe0 sp=0xc000efbe00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000efbfe8 sp=0xc000efbfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 111 [select]: +runtime.gopark(0xc001095fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001095f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001095de8 sp=0xc001095dc8 pc=0x4777ae +runtime.selectgo(0xc001095fb0, 0xc001095f70, 0x1065963d?, 0x0, 0xc000c70150?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001095f38 sp=0xc001095de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001095fe0 sp=0xc001095f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001095fe8 sp=0xc001095fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 140 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 112 [select]: +runtime.gopark(0xc001094fb0?, 0x2?, 0xb0?, 0x4d?, 0xc001094ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001094d58 sp=0xc001094d38 pc=0x4777ae +runtime.selectgo(0xc001094fb0, 0xc001094ee0, 0x392effa?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001094ea8 sp=0xc001094d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001094fe0 sp=0xc001094ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001094fe8 sp=0xc001094fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 140 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 113 [chan receive]: +runtime.gopark(0xc000ef6e30?, 0x1?, 0x1?, 0x0?, 0x3?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000ef6d60 sp=0xc000ef6d40 pc=0x4777ae +runtime.chanrecv(0xc001089380, 0xc000ef6e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc000ef6dd8 sp=0xc000ef6d60 pc=0x441325 +runtime.chanrecv1(0xc00055e1e8?, 0xc000553570?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc000ef6e00 sp=0xc000ef6dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc000ef6fe0 sp=0xc000ef6e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000ef6fe8 sp=0xc000ef6fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 140 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 210 [select]: +runtime.gopark(0xc001131fa0?, 0x3?, 0xf0?, 0x1d?, 0xc001131f3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001495da8 sp=0xc001495d88 pc=0x4777ae +runtime.selectgo(0xc001495fa0, 0xc001131f34, 0xc000ef7f40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001495ef8 sp=0xc001495da8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:184 +0x1b3 fp=0xc001495fe0 sp=0xc001495ef8 pc=0x21557f3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001495fe8 sp=0xc001495fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream in goroutine 140 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:182 +0x24a + +goroutine 196 [select]: +runtime.gopark(0xc000e7f310?, 0x3?, 0xd9?, 0xea?, 0xc000e7f292?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000e7f0f0 sp=0xc000e7f0d0 pc=0x4777ae +runtime.selectgo(0xc000e7f310, 0xc000e7f28c, 0x46216a2?, 0x0, 0x4?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000e7f240 sp=0xc000e7f0f0 pc=0x48916b +github.com/hashicorp/go-plugin.(*grpcStdioServer).StreamStdio(0xc000c6e680, 0x42219c0?, {0x5b4ebb0, 0xc000b6f820}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:61 +0x1e5 fp=0xc000e7f350 sp=0xc000e7f240 pc=0x215bbc5 +github.com/hashicorp/go-plugin/internal/plugin._GRPCStdio_StreamStdio_Handler({0x409e580?, 0xc000c6e680}, {0x5b4a6f0, 0xc000ee6a20}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:185 +0xf7 fp=0xc000e7f3b0 sp=0xc000e7f350 pc=0x2104077 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x409e580, 0xc000c6e680}, {0x5b4ab30, 0xc000d193b0}, 0xc000f1ecc0?, 0x47efca8) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc000e7f400 sp=0xc000e7f3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x409e580, 0xc000c6e680}, {0x5b4ab30, 0xc000d193b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc000e7f490 sp=0xc000e7f400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x409e580, 0xc000c6e680}, {0x5b492f8, 0xc0009ecf00}, 0xc000f1ecc0, 0xc000fe7180) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc000e7f7d0 sp=0xc000e7f490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x409e580, 0xc000c6e680}, {0x5b492f8, 0xc0009ecf00}, 0x3f252a0?, 0xc000b6f5b0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc000e7f848 sp=0xc000e7f7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00094f320, 0xc000c70750, 0x770cba0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc000e7fdb0 sp=0xc000e7f848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00094f320, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc000e7ff28 sp=0xc000e7fdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc000e7ffe0 sp=0xc000e7ff28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000e7ffe8 sp=0xc000e7ffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 189 [select]: +runtime.gopark(0xc000f0e890?, 0x2?, 0x4e?, 0x1a?, 0xc000f0e83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f0e6a8 sp=0xc000f0e688 pc=0x4777ae +runtime.selectgo(0xc000f0e890, 0xc000f0e838, 0x7f429a410b00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000f0e7f8 sp=0xc000f0e6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000b76b90, {0xc000c71bd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc000f0e8c0 sp=0xc000f0e7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000b76b90, {0xc000c71bd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc000f0e958 sp=0xc000f0e8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000c719b0, {0xc000c71bd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc000f0e9a0 sp=0xc000f0e958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc000c719b0}, {0xc000c71bd0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000f0e9e8 sp=0xc000f0e9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000e51440, {0xc000c71bd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc000f0ea58 sp=0xc000f0e9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc000c71bc0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc000f0eb28 sp=0xc000f0ea58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0010b60c0?, 0xc000e51440, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc000f0ec18 sp=0xc000f0eb28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc000c71bc0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0009d60f0}, 0xc000c4a360?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc000f0ecd0 sp=0xc000f0ec18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000c071d0, {0x43dcf60?, 0xc0009d60f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc000f0ef08 sp=0xc000f0ecd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc000c71d10, {0x43dcf60, 0xc0009d60f0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc000f0eff0 sp=0xc000f0ef08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc000b75860, {0x43dcf60, 0xc0009d60f0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc000f0f028 sp=0xc000f0eff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc000c6ede0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc000f0f068 sp=0xc000f0f028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc000c6ede0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc000f0f0a8 sp=0xc000f0f068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc000c6edf0) + :1 +0x46 fp=0xc000f0f0e0 sp=0xc000f0f0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c4a4e0, {0x5b51368, 0xc000c6edf0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc000f0f320 sp=0xc000f0f0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000bfd440, {0x5b51038, 0xc000c6ede0}) + :1 +0xaf fp=0xc000f0f370 sp=0xc000f0f320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000bfd440}, {0x5b4a6f0?, 0xc000b75860}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc000f0f3b0 sp=0xc000f0f370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc000c71d10}, 0xc000dc21b0?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc000f0f400 sp=0xc000f0f3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc000c71d10}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc000f0f490 sp=0xc000f0f400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc000c071d0}, 0xc000dc21b0, 0xc00091ca40) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc000f0f7d0 sp=0xc000f0f490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc000c071d0}, 0x3f252a0?, 0xc000c6ed10?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc000f0f848 sp=0xc000f0f7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc000e51440, 0xc000c4a570, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc000f0fdb0 sp=0xc000f0f848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc000e51440, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc000f0ff28 sp=0xc000f0fdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc000f0ffe0 sp=0xc000f0ff28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f0ffe8 sp=0xc000f0ffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 197 [select]: +runtime.gopark(0xc000d0e890?, 0x2?, 0x4e?, 0x1a?, 0xc000d0e83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d0e6a8 sp=0xc000d0e688 pc=0x4777ae +runtime.selectgo(0xc000d0e890, 0xc000d0e838, 0x7f429a410800?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d0e7f8 sp=0xc000d0e6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0000fc730, {0xc000e394b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc000d0e8c0 sp=0xc000d0e7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0000fc730, {0xc000e394b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc000d0e958 sp=0xc000d0e8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000eb8360, {0xc000e394b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc000d0e9a0 sp=0xc000d0e958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc000eb8360}, {0xc000e394b0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000d0e9e8 sp=0xc000d0e9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00094f440, {0xc000e394b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc000d0ea58 sp=0xc000d0e9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc000e394a0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc000d0eb28 sp=0xc000d0ea58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0010b6080?, 0xc00094f440, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc000d0ec18 sp=0xc000d0eb28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc000e394a0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0009d6000}, 0x13908f0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc000d0ecd0 sp=0xc000d0ec18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc0010a62d0, {0x43dcf60?, 0xc0009d6000}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc000d0ef08 sp=0xc000d0ecd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc000e395f0, {0x43dcf60, 0xc0009d6000}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc000d0eff0 sp=0xc000d0ef08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0009974e0, {0x43dcf60, 0xc0009d6000}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc000d0f028 sp=0xc000d0eff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc000813fc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc000d0f068 sp=0xc000d0f028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc000813fc0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc000d0f0a8 sp=0xc000d0f068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc000813fd0) + :1 +0x46 fp=0xc000d0f0e0 sp=0xc000d0f0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c703c0, {0x5b51368, 0xc000813fd0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc000d0f320 sp=0xc000d0f0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000c6e5e0, {0x5b51038, 0xc000813fc0}) + :1 +0xaf fp=0xc000d0f370 sp=0xc000d0f320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000c6e5e0}, {0x5b4a6f0?, 0xc0009974e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc000d0f3b0 sp=0xc000d0f370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc000e395f0}, 0xc000863080?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc000d0f400 sp=0xc000d0f3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc000e395f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc000d0f490 sp=0xc000d0f400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc0010a62d0}, 0xc000863080, 0xc000dd9380) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc000d0f7d0 sp=0xc000d0f490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc0010a62d0}, 0x3f252a0?, 0xc000813ef0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc000d0f848 sp=0xc000d0f7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00094f440, 0xc000c70450, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc000d0fdb0 sp=0xc000d0f848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00094f440, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc000d0ff28 sp=0xc000d0fdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc000d0ffe0 sp=0xc000d0ff28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d0ffe8 sp=0xc000d0ffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 190 [select]: +runtime.gopark(0xc0012cca68?, 0x2?, 0x0?, 0x0?, 0xc0012cca14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012cc880 sp=0xc0012cc860 pc=0x4777ae +runtime.selectgo(0xc0012cca68, 0xc0012cca10, 0x7f429a410500?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0012cc9d0 sp=0xc0012cc880 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000b76c80, {0xc000d194b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0012cca98 sp=0xc0012cc9d0 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000b76c80, {0xc000d194b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0012ccb30 sp=0xc0012cca98 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000c71b00, {0xc000d194b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0012ccb78 sp=0xc0012ccb30 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc000c71b00}, {0xc000d194b0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0012ccbc0 sp=0xc0012ccb78 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000e51560, {0xc000d194b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0012ccc30 sp=0xc0012ccbc0 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc000d194a0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0012ccd00 sp=0xc0012ccc30 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc000e51560, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0012ccdf0 sp=0xc0012ccd00 pc=0x1171c5c +google.golang.org/grpc.recv(0xc000d194a0, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43dd860, 0xc0016634f0}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0012ccea8 sp=0xc0012ccdf0 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc0009ecff0, {0x43dd860?, 0xc0016634f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0012cd0e0 sp=0xc0012ccea8 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc000d195f0, {0x43dd860, 0xc0016634f0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0012cd1c8 sp=0xc0012cd0e0 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc000ee6a80, {0x43dd860, 0xc0016634f0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0012cd200 sp=0xc0012cd1c8 pc=0x1fd94b2 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamServer).Recv(0xc000b6fe40) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:167 +0x66 fp=0xc0012cd240 sp=0xc0012cd200 pc=0x2102886 +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream(0xc000c4a630, {0x5b512b8, 0xc000b6fe40}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:87 +0x218 fp=0xc0012cd370 sp=0xc0012cd240 pc=0x2154718 +github.com/hashicorp/go-plugin/internal/plugin._GRPCBroker_StartStream_Handler({0x41cb860?, 0xc000c4a630}, {0x5b4a6f0?, 0xc000ee6a80}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:148 +0xb6 fp=0xc0012cd3b0 sp=0xc0012cd370 pc=0x2102716 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41cb860, 0xc000c4a630}, {0x5b4ab30, 0xc000d195f0}, 0xc000f1ed68?, 0x47efc98) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0012cd400 sp=0xc0012cd3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41cb860, 0xc000c4a630}, {0x5b4ab30, 0xc000d195f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0012cd490 sp=0xc0012cd400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41cb860, 0xc000c4a630}, {0x5b492f8, 0xc0009ecff0}, 0xc000f1ed68, 0xc000fe72c0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0012cd7d0 sp=0xc0012cd490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41cb860, 0xc000c4a630}, {0x5b492f8, 0xc0009ecff0}, 0x3f252a0?, 0xc000b6f870?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0012cd848 sp=0xc0012cd7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc000e51560, 0xc000c4a6c0, 0x770cb80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0012cddb0 sp=0xc0012cd848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc000e51560, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0012cdf28 sp=0xc0012cddb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0012cdfe0 sp=0xc0012cdf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012cdfe8 sp=0xc0012cdfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 198 [select]: +runtime.gopark(0xc0010c8a68?, 0x2?, 0x0?, 0x0?, 0xc0010c8a14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010c8880 sp=0xc0010c8860 pc=0x4777ae +runtime.selectgo(0xc0010c8a68, 0xc0010c8a10, 0x7f429a410200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010c89d0 sp=0xc0010c8880 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0000fd130, {0xc000eb8580, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0010c8a98 sp=0xc0010c89d0 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0000fd130, {0xc000eb8580, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0010c8b30 sp=0xc0010c8a98 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000eb84b0, {0xc000eb8580, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0010c8b78 sp=0xc0010c8b30 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc000eb84b0}, {0xc000eb8580, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0010c8bc0 sp=0xc0010c8b78 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00094f560, {0xc000eb8580, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0010c8c30 sp=0xc0010c8bc0 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc000eb8570, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0010c8d00 sp=0xc0010c8c30 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc00094f560, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0010c8df0 sp=0xc0010c8d00 pc=0x1171c5c +google.golang.org/grpc.recv(0xc000eb8570, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43dd860, 0xc0011f25f0}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0010c8ea8 sp=0xc0010c8df0 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000e840f0, {0x43dd860?, 0xc0011f25f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0010c90e0 sp=0xc0010c8ea8 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc000eb86c0, {0x43dd860, 0xc0011f25f0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0010c91c8 sp=0xc0010c90e0 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0005fb2c0, {0x43dd860, 0xc0011f25f0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0010c9200 sp=0xc0010c91c8 pc=0x1fd94b2 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamServer).Recv(0xc000e9c580) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:167 +0x66 fp=0xc0010c9240 sp=0xc0010c9200 pc=0x2102886 +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream(0xc000c70510, {0x5b512b8, 0xc000e9c580}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:87 +0x218 fp=0xc0010c9370 sp=0xc0010c9240 pc=0x2154718 +github.com/hashicorp/go-plugin/internal/plugin._GRPCBroker_StartStream_Handler({0x41cb860?, 0xc000c70510}, {0x5b4a6f0?, 0xc0005fb2c0}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:148 +0xb6 fp=0xc0010c93b0 sp=0xc0010c9370 pc=0x2102716 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41cb860, 0xc000c70510}, {0x5b4ab30, 0xc000eb86c0}, 0xc000a724f8?, 0x47efc98) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0010c9400 sp=0xc0010c93b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41cb860, 0xc000c70510}, {0x5b4ab30, 0xc000eb86c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0010c9490 sp=0xc0010c9400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41cb860, 0xc000c70510}, {0x5b492f8, 0xc000e840f0}, 0xc000a724f8, 0xc0008c4a80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0010c97d0 sp=0xc0010c9490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41cb860, 0xc000c70510}, {0x5b492f8, 0xc000e840f0}, 0x3f252a0?, 0xc000e9c4b0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0010c9848 sp=0xc0010c97d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00094f560, 0xc000c705a0, 0x770cb80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0010c9db0 sp=0xc0010c9848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00094f560, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0010c9f28 sp=0xc0010c9db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0010c9fe0 sp=0xc0010c9f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010c9fe8 sp=0xc0010c9fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 199 [select]: +runtime.gopark(0xc000efcfa0?, 0x3?, 0x0?, 0x4e?, 0xc000efcf3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000efcda8 sp=0xc000efcd88 pc=0x4777ae +runtime.selectgo(0xc000efcfa0, 0xc000efcf34, 0xc000efcf40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000efcef8 sp=0xc000efcda8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:73 +0x1ad fp=0xc000efcfe0 sp=0xc000efcef8 pc=0x2154aad +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000efcfe8 sp=0xc000efcfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream in goroutine 198 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:71 +0x1d1 + +goroutine 211 [select]: +runtime.gopark(0xc000efdfa0?, 0x3?, 0x0?, 0x0?, 0xc000efdf3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000efdda8 sp=0xc000efdd88 pc=0x4777ae +runtime.selectgo(0xc000efdfa0, 0xc000efdf34, 0xc000efdf40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000efdef8 sp=0xc000efdda8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:73 +0x1ad fp=0xc000efdfe0 sp=0xc000efdef8 pc=0x2154aad +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000efdfe8 sp=0xc000efdfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream in goroutine 190 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:71 +0x1d1 + +goroutine 191 [select]: +runtime.gopark(0xc001096fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001096f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001096de8 sp=0xc001096dc8 pc=0x4777ae +runtime.selectgo(0xc001096fb0, 0xc001096f70, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001096f38 sp=0xc001096de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001096fe0 sp=0xc001096f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001096fe8 sp=0xc001096fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 192 [select]: +runtime.gopark(0xc0010aefb0?, 0x2?, 0xb0?, 0xed?, 0xc0010aeee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010aed58 sp=0xc0010aed38 pc=0x4777ae +runtime.selectgo(0xc0010aefb0, 0xc0010aeee0, 0x448855?, 0x0, 0x4166b20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010aeea8 sp=0xc0010aed58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0010aefe0 sp=0xc0010aeea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010aefe8 sp=0xc0010aefe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 193 [chan receive]: +runtime.gopark(0x7700000000000000?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001097d60 sp=0xc001097d40 pc=0x4777ae +runtime.chanrecv(0xc000c6b1a0, 0xc001097e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001097dd8 sp=0xc001097d60 pc=0x441325 +runtime.chanrecv1(0x28?, 0x14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001097e00 sp=0xc001097dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001097fe0 sp=0xc001097e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001097fe8 sp=0xc001097fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 626 [chan receive]: +runtime.gopark(0xc000912b88?, 0x4b05a9?, 0x28?, 0x2f?, 0x5d54ac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001242ee8 sp=0xc001242ec8 pc=0x4777ae +runtime.chanrecv(0xc000f26540, 0xc001242fa0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001242f60 sp=0xc001242ee8 pc=0x441325 +runtime.chanrecv2(0x4ad421?, 0xc001242fd0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc001242f88 sp=0xc001242f60 pc=0x440f92 +google.golang.org/grpc.(*Server).serverWorker(0xc001ebf4a0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:593 +0x85 fp=0xc001242fc0 sp=0xc001242f88 pc=0x11744a5 +google.golang.org/grpc.(*Server).initServerWorkers.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0x34 fp=0xc001242fe0 sp=0xc001242fc0 pc=0x1174994 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001242fe8 sp=0xc001242fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).initServerWorkers in goroutine 559 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0xa5 + +goroutine 201 [select]: +runtime.gopark(0xc0010abfb0?, 0x2?, 0xb4?, 0x1b?, 0xc0010abf74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010abde8 sp=0xc0010abdc8 pc=0x4777ae +runtime.selectgo(0xc0010abfb0, 0xc0010abf70, 0x2154a5f?, 0x0, 0xc000e9c580?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010abf38 sp=0xc0010abde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0010abfe0 sp=0xc0010abf38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010abfe8 sp=0xc0010abfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 202 [select]: +runtime.gopark(0xc0010acfb0?, 0x2?, 0xb0?, 0xcd?, 0xc0010acee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010acd58 sp=0xc0010acd38 pc=0x4777ae +runtime.selectgo(0xc0010acfb0, 0xc0010acee0, 0x280487a?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010acea8 sp=0xc0010acd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0010acfe0 sp=0xc0010acea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010acfe8 sp=0xc0010acfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 203 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010aad60 sp=0xc0010aad40 pc=0x4777ae +runtime.chanrecv(0xc000c9da40, 0xc0010aae60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0010aadd8 sp=0xc0010aad60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0010aae00 sp=0xc0010aadd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc0010aafe0 sp=0xc0010aae00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010aafe8 sp=0xc0010aafe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 226 [select]: +runtime.gopark(0xc000f02890?, 0x2?, 0x4e?, 0x1a?, 0xc000f0283c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f026a8 sp=0xc000f02688 pc=0x4777ae +runtime.selectgo(0xc000f02890, 0xc000f02838, 0x7f429a416200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000f027f8 sp=0xc000f026a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000b77b30, {0xc00110a430, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc000f028c0 sp=0xc000f027f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000b77b30, {0xc00110a430, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc000f02958 sp=0xc000f028c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00110a360, {0xc00110a430, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc000f029a0 sp=0xc000f02958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc00110a360}, {0xc00110a430, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000f029e8 sp=0xc000f029a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000e51d40, {0xc00110a430, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc000f02a58 sp=0xc000f029e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc00110a420, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc000f02b28 sp=0xc000f02a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001109440?, 0xc000e51d40, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc000f02c18 sp=0xc000f02b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc00110a420, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc000b77f90}, 0xc000c70240?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc000f02cd0 sp=0xc000f02c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000c072c0, {0x43dcf60?, 0xc000b77f90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc000f02f08 sp=0xc000f02cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc00110a570, {0x43dcf60, 0xc000b77f90}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc000f02ff0 sp=0xc000f02f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc000b75a80, {0x43dcf60, 0xc000b77f90}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc000f03028 sp=0xc000f02ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc000c6f0b0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc000f03068 sp=0xc000f03028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc000c6f0b0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc000f030a8 sp=0xc000f03068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc000c6f0c0) + :1 +0x46 fp=0xc000f030e0 sp=0xc000f030a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c703c0, {0x5b51368, 0xc000c6f0c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc000f03320 sp=0xc000f030e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000c6e5e0, {0x5b51038, 0xc000c6f0b0}) + :1 +0xaf fp=0xc000f03370 sp=0xc000f03320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000c6e5e0}, {0x5b4a6f0?, 0xc000b75a80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc000f033b0 sp=0xc000f03370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc00110a570}, 0xc000dc24c8?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc000f03400 sp=0xc000f033b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc00110a570}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc000f03490 sp=0xc000f03400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc000c072c0}, 0xc000dc24c8, 0xc001108840) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc000f037d0 sp=0xc000f03490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc000c072c0}, 0x3f252a0?, 0xc000c6efe0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc000f03848 sp=0xc000f037d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc000e51d40, 0xc000c70450, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc000f03db0 sp=0xc000f03848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc000e51d40, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc000f03f28 sp=0xc000f03db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc000f03fe0 sp=0xc000f03f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f03fe8 sp=0xc000f03fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 627 [chan receive]: +runtime.gopark(0x0?, 0x4?, 0x51?, 0x80?, 0xc001f16898?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010afee8 sp=0xc0010afec8 pc=0x4777ae +runtime.chanrecv(0xc000f26540, 0xc0010affa0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0010aff60 sp=0xc0010afee8 pc=0x441325 +runtime.chanrecv2(0x1?, 0x1?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0010aff88 sp=0xc0010aff60 pc=0x440f92 +google.golang.org/grpc.(*Server).serverWorker(0xc001ebf4a0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:593 +0x85 fp=0xc0010affc0 sp=0xc0010aff88 pc=0x11744a5 +google.golang.org/grpc.(*Server).initServerWorkers.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0x34 fp=0xc0010affe0 sp=0xc0010affc0 pc=0x1174994 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010affe8 sp=0xc0010affe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).initServerWorkers in goroutine 559 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0xa5 + +goroutine 204 [select]: +runtime.gopark(0xc000f06890?, 0x2?, 0x98?, 0x67?, 0xc000f0683c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f066a8 sp=0xc000f06688 pc=0x4777ae +runtime.selectgo(0xc000f06890, 0xc000f06838, 0x7f429a415c00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000f067f8 sp=0xc000f066a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0008302d0, {0xc000eb8c40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc000f068c0 sp=0xc000f067f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0008302d0, {0xc000eb8c40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc000f06958 sp=0xc000f068c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000eb8b70, {0xc000eb8c40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc000f069a0 sp=0xc000f06958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc000eb8b70}, {0xc000eb8c40, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000f069e8 sp=0xc000f069a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00094f9e0, {0xc000eb8c40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc000f06a58 sp=0xc000f069e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc000eb8c30, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc000f06b28 sp=0xc000f06a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc000ed0a80?, 0xc00094f9e0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc000f06c18 sp=0xc000f06b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc000eb8c30, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc000830690}, 0xc000c4a360?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc000f06cd0 sp=0xc000f06c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000e841e0, {0x43dcf60?, 0xc000830690}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc000f06f08 sp=0xc000f06cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc000eb8d80, {0x43dcf60, 0xc000830690}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc000f06ff0 sp=0xc000f06f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0005fb480, {0x43dcf60, 0xc000830690}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc000f07028 sp=0xc000f06ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc000e9c8b0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc000f07068 sp=0xc000f07028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc000e9c8b0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc000f070a8 sp=0xc000f07068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc000e9c8c0) + :1 +0x46 fp=0xc000f070e0 sp=0xc000f070a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c4a4e0, {0x5b51368, 0xc000e9c8c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc000f07320 sp=0xc000f070e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000bfd440, {0x5b51038, 0xc000e9c8b0}) + :1 +0xaf fp=0xc000f07370 sp=0xc000f07320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000bfd440}, {0x5b4a6f0?, 0xc0005fb480}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc000f073b0 sp=0xc000f07370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc000eb8d80}, 0xc000a726f0?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc000f07400 sp=0xc000f073b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc000eb8d80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc000f07490 sp=0xc000f07400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc000e841e0}, 0xc000a726f0, 0xc000ed0080) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc000f077d0 sp=0xc000f07490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc000e841e0}, 0x3f252a0?, 0xc000e9c7e0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc000f07848 sp=0xc000f077d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc00094f9e0, 0xc000c4a570, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc000f07db0 sp=0xc000f07848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc00094f9e0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc000f07f28 sp=0xc000f07db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc000f07fe0 sp=0xc000f07f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f07fe8 sp=0xc000f07fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 513 [select]: +runtime.gopark(0xc001244f40?, 0x3?, 0xfd?, 0xfd?, 0xc001244d2a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001244b60 sp=0xc001244b40 pc=0x4777ae +runtime.selectgo(0xc001244f40, 0xc001244d24, 0xc001244f70?, 0x0, 0x48e2a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001244cb0 sp=0xc001244b60 pc=0x48916b +github.com/rancher/opni/pkg/health.(*Monitor).WatchHealthStatus.func1() + /home/ec2-user/opni/pkg/health/monitor.go:166 +0x68e fp=0xc001244fe0 sp=0xc001244cb0 pc=0x207082e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001244fe8 sp=0xc001244fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/health.(*Monitor).WatchHealthStatus in goroutine 573 + /home/ec2-user/opni/pkg/health/monitor.go:150 +0x8da + +goroutine 236 [select]: +runtime.gopark(0xc001134890?, 0x2?, 0x98?, 0x47?, 0xc00113483c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011346a8 sp=0xc001134688 pc=0x4777ae +runtime.selectgo(0xc001134890, 0xc001134838, 0x7f429a415600?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011347f8 sp=0xc0011346a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00099b950, {0xc00110b780, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0011348c0 sp=0xc0011347f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00099b950, {0xc00110b780, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001134958 sp=0xc0011348c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00110af30, {0xc00110b780, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0011349a0 sp=0xc001134958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc00110af30}, {0xc00110b780, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0011349e8 sp=0xc0011349a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00110e6c0, {0xc00110b780, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001134a58 sp=0xc0011349e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc00110b770, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001134b28 sp=0xc001134a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc00112b000?, 0xc00110e6c0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001134c18 sp=0xc001134b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc00110b770, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc00010f6d0}, 0xc000c70240?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001134cd0 sp=0xc001134c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000c073b0, {0x43dcf60?, 0xc00010f6d0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001134f08 sp=0xc001134cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc00110b8c0, {0x43dcf60, 0xc00010f6d0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001134ff0 sp=0xc001134f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc000b75f20, {0x43dcf60, 0xc00010f6d0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001135028 sp=0xc001134ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc000c6f940) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001135068 sp=0xc001135028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc000c6f940?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0011350a8 sp=0xc001135068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc000c6f950) + :1 +0x46 fp=0xc0011350e0 sp=0xc0011350a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c703c0, {0x5b51368, 0xc000c6f950}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001135320 sp=0xc0011350e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000c6e5e0, {0x5b51038, 0xc000c6f940}) + :1 +0xaf fp=0xc001135370 sp=0xc001135320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000c6e5e0}, {0x5b4a6f0?, 0xc000b75f20}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0011353b0 sp=0xc001135370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc00110b8c0}, 0xc000dc2c30?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001135400 sp=0xc0011353b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc00110b8c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001135490 sp=0xc001135400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc000c073b0}, 0xc000dc2c30, 0xc00112a400) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0011357d0 sp=0xc001135490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc000c073b0}, 0x3f252a0?, 0xc000c6f870?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001135848 sp=0xc0011357d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00110e6c0, 0xc000c70450, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001135db0 sp=0xc001135848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00110e6c0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001135f28 sp=0xc001135db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001135fe0 sp=0xc001135f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001135fe8 sp=0xc001135fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 216 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x20?, 0x75?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001187cd0 sp=0xc001187cb0 pc=0x4777ae +runtime.chanrecv(0xc000fa56e0, 0xc001187de0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001187d48 sp=0xc001187cd0 pc=0x441325 +runtime.chanrecv1(0xc001187df0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001187d70 sp=0xc001187d48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc001187f30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc001187e00 sp=0xc001187d70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc000bdd540, 0x420c60?, 0xc000d19770) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc001187fa0 sp=0xc001187e00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc001187fe0 sp=0xc001187fa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001187fe8 sp=0xc001187fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 213 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 230 [select]: +runtime.gopark(0xc001110fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001110f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001110de8 sp=0xc001110dc8 pc=0x4777ae +runtime.selectgo(0xc001110fb0, 0xc001110f70, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001110f38 sp=0xc001110de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001110fe0 sp=0xc001110f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001110fe8 sp=0xc001110fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 231 [select]: +runtime.gopark(0xc001116fb0?, 0x2?, 0xb0?, 0x6d?, 0xc001116ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001116d58 sp=0xc001116d38 pc=0x4777ae +runtime.selectgo(0xc001116fb0, 0xc001116ee0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001116ea8 sp=0xc001116d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001116fe0 sp=0xc001116ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001116fe8 sp=0xc001116fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 232 [chan receive]: +runtime.gopark(0x5000000000000000?, 0x0?, 0x1?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001098d60 sp=0xc001098d40 pc=0x4777ae +runtime.chanrecv(0xc000c6b500, 0xc001098e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001098dd8 sp=0xc001098d60 pc=0x441325 +runtime.chanrecv1(0x28?, 0x14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001098e00 sp=0xc001098dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001098fe0 sp=0xc001098e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001098fe8 sp=0xc001098fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 628 [chan receive]: +runtime.gopark(0xc000912b80?, 0x2182b94?, 0x78?, 0xcf?, 0x24c59ff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012acee8 sp=0xc0012acec8 pc=0x4777ae +runtime.chanrecv(0xc000f26540, 0xc0012acfa0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0012acf60 sp=0xc0012acee8 pc=0x441325 +runtime.chanrecv2(0x4ad421?, 0xc0012acfd0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0012acf88 sp=0xc0012acf60 pc=0x440f92 +google.golang.org/grpc.(*Server).serverWorker(0xc001ebf4a0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:593 +0x85 fp=0xc0012acfc0 sp=0xc0012acf88 pc=0x11744a5 +google.golang.org/grpc.(*Server).initServerWorkers.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0x34 fp=0xc0012acfe0 sp=0xc0012acfc0 pc=0x1174994 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012acfe8 sp=0xc0012acfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).initServerWorkers in goroutine 559 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0xa5 + +goroutine 262 [chan receive]: +runtime.gopark(0x4be6c0?, 0xc000c4b2c0?, 0x20?, 0xe7?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001146f08 sp=0xc001146ee8 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001146f80 sp=0xc001146f08 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0x5b11a20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001146fa8 sp=0xc001146f80 pc=0x440f72 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseNodeManagerClient(0xc0005a8c80, {0x5b11a20, 0xc000fe12f0}) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:222 +0x93 fp=0xc001146fd0 sp=0xc001146fa8 pc=0x3972293 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseNodeManagerClient(0xc00091c300, {0x4b05a9?, 0xc0012130f0?}, 0xc0010bb410) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:113 +0x18d fp=0xc001147098 sp=0xc001146fd0 pc=0x21a68cd +github.com/rancher/opni/pkg/plugins/apis/system._System_UseNodeManagerClient_Handler.func1({0x5b426a8, 0xc0010bb4d0}, {0x4278de0?, 0xc0010bb410}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:169 +0x83 fp=0xc0011470f0 sp=0xc001147098 pc=0x21ac443 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc0010bb4d0}, {0x4278de0, 0xc0010bb410}, 0xc000997aa0, 0xc000863e78) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc001147148 sp=0xc0011470f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc0010bb4d0}, {0x4278de0, 0xc0010bb410}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc0011471e8 sp=0xc001147148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc0010bb3e0}, {0x4278de0, 0xc0010bb410}, 0xc000997aa0, 0xc0010b6b80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc0011476b8 sp=0xc0011471e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc0010bb3e0}, {0x4278de0, 0xc0010bb410}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc001147740 sp=0xc0011476b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseNodeManagerClient_Handler({0x4278ea0?, 0xc00091c300}, {0x5b426a8, 0xc0010bb3e0}, 0xc0004b70a0, 0xc000b75440) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:171 +0x1c4 fp=0xc0011477d0 sp=0xc001147740 pc=0x21ac2a4 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc000f8b8c0, 0xc000c70f60, 0x771ecb8, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc001147db0 sp=0xc0011477d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc000f8b8c0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc001147f28 sp=0xc001147db0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001147fe0 sp=0xc001147f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001147fe8 sp=0xc001147fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 296 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0xe0?, 0x7b?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010a1cd0 sp=0xc0010a1cb0 pc=0x4777ae +runtime.chanrecv(0xc0012485a0, 0xc0010a1de0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0010a1d48 sp=0xc0010a1cd0 pc=0x441325 +runtime.chanrecv1(0xc0010a1df0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0010a1d70 sp=0xc0010a1d48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc0010a1f30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc0010a1e00 sp=0xc0010a1d70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc000bdd540, 0x0?, 0xc000ff95c0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc0010a1fa0 sp=0xc0010a1e00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc0010a1fe0 sp=0xc0010a1fa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010a1fe8 sp=0xc0010a1fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 294 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 206 [select]: +runtime.gopark(0xc001194fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001194f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001194de8 sp=0xc001194dc8 pc=0x4777ae +runtime.selectgo(0xc001194fb0, 0xc001194f70, 0x2154a5f?, 0x0, 0xc000e9c580?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001194f38 sp=0xc001194de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001194fe0 sp=0xc001194f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001194fe8 sp=0xc001194fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 207 [select]: +runtime.gopark(0xc001193fb0?, 0x2?, 0xb0?, 0x3d?, 0xc001193ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001193d58 sp=0xc001193d38 pc=0x4777ae +runtime.selectgo(0xc001193fb0, 0xc001193ee0, 0x472ddb?, 0x0, 0x2166b62?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001193ea8 sp=0xc001193d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001193fe0 sp=0xc001193ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001193fe8 sp=0xc001193fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 208 [chan receive]: +runtime.gopark(0x7700000000000000?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010add60 sp=0xc0010add40 pc=0x4777ae +runtime.chanrecv(0xc000c9dda0, 0xc0010ade60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0010addd8 sp=0xc0010add60 pc=0x441325 +runtime.chanrecv1(0x28?, 0x14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0010ade00 sp=0xc0010addd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc0010adfe0 sp=0xc0010ade00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010adfe8 sp=0xc0010adfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 722 [chan receive]: +runtime.gopark(0x0?, 0x4b05a9?, 0x50?, 0x8e?, 0x4bff4a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019f8e18 sp=0xc0019f8df8 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0019f8e90 sp=0xc0019f8e18 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0x7f429a441700?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0019f8eb8 sp=0xc0019f8e90 pc=0x440f72 +github.com/rancher/opni/pkg/gateway.NewGateway.func5() + /home/ec2-user/opni/pkg/gateway/gateway.go:308 +0x5c fp=0xc0019f8f50 sp=0xc0019f8eb8 pc=0x22aea9c +github.com/rancher/opni/pkg/util/waitctx.restrictive.Go.func1() + /home/ec2-user/opni/pkg/util/waitctx/context.go:162 +0x93 fp=0xc0019f8fe0 sp=0xc0019f8f50 pc=0x178df73 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0019f8fe8 sp=0xc0019f8fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/util/waitctx.restrictive.Go in goroutine 609 + /home/ec2-user/opni/pkg/util/waitctx/context.go:160 +0xeb + +goroutine 217 [select]: +runtime.gopark(0xc00120f448?, 0x2?, 0xd0?, 0xf2?, 0xc00120f3fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00120f278 sp=0xc00120f258 pc=0x4777ae +runtime.selectgo(0xc00120f448, 0xc00120f3f8, 0xc000b6efe8?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00120f3c8 sp=0xc00120f278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000926a20) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc00120f478 sp=0xc00120f3c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00094cf70, {0x42219c0?, 0xc000d19d10}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc00120f6d8 sp=0xc00120f478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000926894?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc00120f738 sp=0xc00120f6d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc0009267e0, 0xc00120f8f0, 0xc00120f8c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc00120f7e0 sp=0xc00120f738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0009267e0, {0x42219c0, 0xc000d19d10}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc00120f920 sp=0xc00120f7e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc000d19dd0}, {0x4661da0, 0x1f}, {0x4278de0, 0xc000d19ce0}, {0x42219c0, 0xc000d19d10}, 0x0?, {0xc0005fac80, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc00120f9c0 sp=0xc00120f920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc000bdcaa0}, {0x4661da0, 0x1f}, {0x4278de0, 0xc000d19ce0}, {0x42219c0, 0xc000d19d10}, 0xc0009c7400, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc00120fd30 sp=0xc00120f9c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc0009c7400, {0x5b426e0, 0xc000bdcaa0}, {0x4661da0, 0x1f}, {0x4278de0, 0xc000d19ce0}, {0x42219c0, 0xc000d19d10}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc00120fe58 sp=0xc00120fd30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseManagementAPI(0xc000c6f160, {0x5b426e0, 0xc000bdcaa0}, 0x0?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:51 +0xee fp=0xc00120fee8 sp=0xc00120fe58 pc=0x21ab66e +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeManagementAPI.func2(0x1) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:190 +0xf2 fp=0xc00120ff58 sp=0xc00120fee8 pc=0x21a7f32 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc00120ffe0 sp=0xc00120ff58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00120ffe8 sp=0xc00120ffe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 213 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 286 [select]: +runtime.gopark(0xc0013f5e48?, 0x2?, 0x0?, 0x0?, 0xc0013f5dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f0dc30 sp=0xc001f0dc10 pc=0x4777ae +runtime.selectgo(0xc001f0de48, 0xc0013f5dc8, 0xc0013f5dd8?, 0x0, 0xc00125e000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001f0dd80 sp=0xc001f0dc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00139c190, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001f0de78 sp=0xc001f0dd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000573030) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001f0df30 sp=0xc001f0de78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc001f0dfe0 sp=0xc001f0df30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f0dfe8 sp=0xc001f0dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 304 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 213 [select]: +runtime.gopark(0xc001113f28?, 0x2?, 0xb4?, 0x1b?, 0xc001113ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001113d30 sp=0xc001113d10 pc=0x4777ae +runtime.selectgo(0xc001113f28, 0xc001113eb8, 0x0?, 0x0, 0xc001113f20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001113e80 sp=0xc001113d30 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc00110a780, 0xc000f1ee28, 0xc000b6ffc0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001113f58 sp=0xc001113e80 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeManagementAPI(0xc00110a780, {0x5b6f770?, 0xc000d8a000}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:185 +0x10a fp=0xc001113f98 sp=0xc001113f58 pc=0x21a7e0a +github.com/rancher/opni/pkg/management.NewServer.func1.2() + /home/ec2-user/opni/pkg/management/server.go:156 +0x62 fp=0xc001113fe0 sp=0xc001113f98 pc=0x231c0c2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001113fe8 sp=0xc001113fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.NewServer.func1 in goroutine 212 + /home/ec2-user/opni/pkg/management/server.go:156 +0x137 + +goroutine 214 [select]: +runtime.gopark(0xc001111f28?, 0x2?, 0xb4?, 0x1b?, 0xc001111ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001111d30 sp=0xc001111d10 pc=0x4777ae +runtime.selectgo(0xc001111f28, 0xc001111eb8, 0x0?, 0x0, 0xc001111f20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001111e80 sp=0xc001111d30 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc00110a780, 0xc000f1ee40, 0xc000fe0020) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001111f58 sp=0xc001111e80 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeNodeManagerServer(0xc00110a780, {0x5b21b70?, 0xc000ef5240}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:198 +0x10a fp=0xc001111f98 sp=0xc001111f58 pc=0x21a810a +github.com/rancher/opni/pkg/management.NewServer.func1.3() + /home/ec2-user/opni/pkg/management/server.go:158 +0x62 fp=0xc001111fe0 sp=0xc001111f98 pc=0x231c022 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001111fe8 sp=0xc001111fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.NewServer.func1 in goroutine 212 + /home/ec2-user/opni/pkg/management/server.go:158 +0x245 + +goroutine 1192 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00172d9a8 sp=0xc00172d988 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00172d9e0 sp=0xc00172d9a8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169d868, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00172da00 sp=0xc00172d9e0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc002083b20, 0x448801?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00172da48 sp=0xc00172da00 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc002083b00) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc00172db38 sp=0xc00172da48 pc=0x513f65 +net.(*netFD).accept(0xc002083b00) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc00172dc30 sp=0xc00172db38 pc=0x8967be +net.(*TCPListener).accept(0xc00127d0a0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc00172dc78 sp=0xc00172dc30 pc=0x8c383e +net.(*TCPListener).Accept(0xc00127d0a0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc00172dce8 sp=0xc00172dc78 pc=0x8c1ba5 +net/http.(*onceCloseListener).Accept(0xc00224ec90) + :1 +0x44 fp=0xc00172dd20 sp=0xc00172dce8 pc=0xab5544 +net/http.(*Server).Serve(0xc001388960, {0x5b2a6d0, 0xc00127d0a0}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc00172df18 sp=0xc00172dd20 pc=0xa7859f +net/http.(*Server).ListenAndServe(0xc001388960) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:2985 +0xbd fp=0xc00172df70 sp=0xc00172df18 pc=0xa77f9d +github.com/rancher/opni/pkg/management.(*Server).listenAndServeHttp.func2() + /home/ec2-user/opni/pkg/management/server.go:246 +0x2f fp=0xc00172dfa0 sp=0xc00172df70 pc=0x231edef +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc00172dfe0 sp=0xc00172dfa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00172dfe8 sp=0xc00172dfe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1144 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 259 [select]: +runtime.gopark(0xc0010b0fb0?, 0x2?, 0x0?, 0xe?, 0xc0010b0f2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010b0da8 sp=0xc0010b0d88 pc=0x4777ae +runtime.selectgo(0xc0010b0fb0, 0xc0010b0f28, 0x0?, 0x0, 0x13?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010b0ef8 sp=0xc0010b0da8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc0010b0fe0 sp=0xc0010b0ef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010b0fe8 sp=0xc0010b0fe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 258 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 218 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x60?, 0x72?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001205cd0 sp=0xc001205cb0 pc=0x4777ae +runtime.chanrecv(0xc000fa54a0, 0xc001205de0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001205d48 sp=0xc001205cd0 pc=0x441325 +runtime.chanrecv1(0xc001205df0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001205d70 sp=0xc001205d48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc001205f30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc001205e00 sp=0xc001205d70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc000bdd540, 0x2?, 0xc000d197d0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc001205fa0 sp=0xc001205e00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc001205fe0 sp=0xc001205fa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001205fe8 sp=0xc001205fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 214 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 219 [select]: +runtime.gopark(0xc001221448?, 0x2?, 0xd0?, 0x12?, 0xc0012213fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001221278 sp=0xc001221258 pc=0x4777ae +runtime.selectgo(0xc001221448, 0xc0012213f8, 0xc000b6efe8?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0012213c8 sp=0xc001221278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000927440) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001221478 sp=0xc0012213c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00094d380, {0x42219c0?, 0xc000ff8570}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0012216d8 sp=0xc001221478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000926bf4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001221738 sp=0xc0012216d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc000926b40, 0xc0012218f0, 0xc0012218c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc0012217e0 sp=0xc001221738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc000926b40, {0x42219c0, 0xc000ff8570}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001221920 sp=0xc0012217e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc000ff8630}, {0x4673e80, 0x23}, {0x4278de0, 0xc000ff8540}, {0x42219c0, 0xc000ff8570}, 0x0?, {0xc0005fac80, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc0012219c0 sp=0xc001221920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc000bdcaa0}, {0x4673e80, 0x23}, {0x4278de0, 0xc000ff8540}, {0x42219c0, 0xc000ff8570}, 0xc0009c7400, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc001221d30 sp=0xc0012219c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc0009c7400, {0x5b426e0, 0xc000bdcaa0}, {0x4673e80, 0x23}, {0x4278de0, 0xc000ff8540}, {0x42219c0, 0xc000ff8570}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc001221e58 sp=0xc001221d30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseNodeManagerClient(0xc000c6f160, {0x5b426e0, 0xc000bdcaa0}, 0xc000b80f50?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:60 +0xee fp=0xc001221ee8 sp=0xc001221e58 pc=0x21ab7ee +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeNodeManagerServer.func2(0x2) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:203 +0xf2 fp=0xc001221f58 sp=0xc001221ee8 pc=0x21a8232 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc001221fe0 sp=0xc001221f58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001221fe8 sp=0xc001221fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 214 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 242 [select]: +runtime.gopark(0xc0011a1fb0?, 0x2?, 0xb4?, 0x1b?, 0xc0011a1f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011a1de8 sp=0xc0011a1dc8 pc=0x4777ae +runtime.selectgo(0xc0011a1fb0, 0xc0011a1f70, 0xc000a53a40?, 0x0, 0xc000c9de00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011a1f38 sp=0xc0011a1de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0011a1fe0 sp=0xc0011a1f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011a1fe8 sp=0xc0011a1fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 209 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 243 [select]: +runtime.gopark(0xc0011a0fb0?, 0x2?, 0x0?, 0x0?, 0xc0011a0ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011a0d58 sp=0xc0011a0d38 pc=0x4777ae +runtime.selectgo(0xc0011a0fb0, 0xc0011a0ee0, 0x215759f?, 0x0, 0x2155d2b?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011a0ea8 sp=0xc0011a0d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0011a0fe0 sp=0xc0011a0ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011a0fe8 sp=0xc0011a0fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 209 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 244 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0xe0?, 0x5d?, 0x22c5a46?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001195d60 sp=0xc001195d40 pc=0x4777ae +runtime.chanrecv(0xc00119a0c0, 0xc001195e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001195dd8 sp=0xc001195d60 pc=0x441325 +runtime.chanrecv1(0xc000e9c970?, 0xc00070f822?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001195e00 sp=0xc001195dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001195fe0 sp=0xc001195e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001195fe8 sp=0xc001195fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 209 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 245 [select]: +runtime.gopark(0xc001224890?, 0x2?, 0x98?, 0x47?, 0xc00122483c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012246a8 sp=0xc001224688 pc=0x4777ae +runtime.selectgo(0xc001224890, 0xc001224838, 0x7f429a419200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0012247f8 sp=0xc0012246a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000831310, {0xc000ff8ac0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0012248c0 sp=0xc0012247f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000831310, {0xc000ff8ac0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001224958 sp=0xc0012248c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000eb96b0, {0xc000ff8ac0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0012249a0 sp=0xc001224958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc000eb96b0}, {0xc000ff8ac0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0012249e8 sp=0xc0012249a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001196240, {0xc000ff8ac0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001224a58 sp=0xc0012249e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc000ff8ab0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001224b28 sp=0xc001224a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001236580?, 0xc001196240, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001224c18 sp=0xc001224b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc000ff8ab0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0007c01e0}, 0xc000c4a360?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001224cd0 sp=0xc001224c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001234000, {0x43dcf60?, 0xc0007c01e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001224f08 sp=0xc001224cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc000ff8c00, {0x43dcf60, 0xc0007c01e0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001224ff0 sp=0xc001224f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc000ee75e0, {0x43dcf60, 0xc0007c01e0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001225028 sp=0xc001224ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc000fe0b60) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001225068 sp=0xc001225028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc000fe0b60?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0012250a8 sp=0xc001225068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc000fe0b70) + :1 +0x46 fp=0xc0012250e0 sp=0xc0012250a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c4a4e0, {0x5b51368, 0xc000fe0b70}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001225320 sp=0xc0012250e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000bfd440, {0x5b51038, 0xc000fe0b60}) + :1 +0xaf fp=0xc001225370 sp=0xc001225320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000bfd440}, {0x5b4a6f0?, 0xc000ee75e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0012253b0 sp=0xc001225370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc000ff8c00}, 0xc000f1f4b8?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001225400 sp=0xc0012253b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc000ff8c00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001225490 sp=0xc001225400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc001234000}, 0xc000f1f4b8, 0xc000fe7b80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0012257d0 sp=0xc001225490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc001234000}, 0x3f252a0?, 0xc000fe0a90?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001225848 sp=0xc0012257d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc001196240, 0xc000c4a570, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001225db0 sp=0xc001225848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc001196240, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001225f28 sp=0xc001225db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001225fe0 sp=0xc001225f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001225fe8 sp=0xc001225fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 246 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x20?, 0xe7?, 0x30?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00114af98 sp=0xc00114af78 pc=0x4777ae +runtime.chanrecv(0xc0011a46c0, 0xc0011c5128, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00114b010 sp=0xc00114af98 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00114b038 sp=0xc00114b010 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/stream.(*gatewayStreamExtensionServerImpl).ConnectInternal(0xc000854620, {0x5b50ab0, 0xc000e9ce00}) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/stream/plugin_gateway.go:208 +0xa4e fp=0xc00114b370 sp=0xc00114b038 pc=0x262088e +github.com/rancher/opni/pkg/plugins/apis/apiextensions._StreamAPIExtension_ConnectInternal_Handler({0x4229080?, 0xc000854620}, {0x5b4a6f0?, 0xc0005fb7c0}) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/apiextensions_grpc.pb.go:284 +0xb6 fp=0xc00114b3b0 sp=0xc00114b370 pc=0x218ded6 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x4229080, 0xc000854620}, {0x5b4ab30, 0xc000eb99e0}, 0xc000a72c30?, 0x47ee568) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc00114b400 sp=0xc00114b3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x4229080, 0xc000854620}, {0x5b4ab30, 0xc000eb99e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc00114b490 sp=0xc00114b400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4229080, 0xc000854620}, {0x5b492f8, 0xc000e842d0}, 0xc000a72c30, 0xc000ed1540) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc00114b7d0 sp=0xc00114b490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x4229080, 0xc000854620}, {0x5b492f8, 0xc000e842d0}, 0x3f252a0?, 0xc000e9cd30?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc00114b848 sp=0xc00114b7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc001196360, 0xc000c4a9c0, 0x770c840, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc00114bdb0 sp=0xc00114b848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc001196360, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc00114bf28 sp=0xc00114bdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc00114bfe0 sp=0xc00114bf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00114bfe8 sp=0xc00114bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 239 [select]: +runtime.gopark(0xc001203f70?, 0x2?, 0xb4?, 0x1b?, 0xc001203ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001203d20 sp=0xc001203d00 pc=0x4777ae +runtime.selectgo(0xc001203f70, 0xc001203eb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001203e70 sp=0xc001203d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0009d2100, {0x5b426e0, 0xc00010f810}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001203fa0 sp=0xc001203e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001203fe0 sp=0xc001203fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001203fe8 sp=0xc001203fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 237 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 177 [chan receive]: +runtime.gopark(0x4b05a9?, 0xc0019c6e68?, 0x6d?, 0x2d?, 0xc0001277a0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001130dd8 sp=0xc001130db8 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001130e50 sp=0xc001130dd8 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0x5b6f168?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001130e78 sp=0xc001130e50 pc=0x440f72 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI(0xc0005a8c80, {0x5b6f168, 0xc000fe1300}) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:92 +0x619 fp=0xc001130fa0 sp=0xc001130e78 pc=0x396f019 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseManagementAPI(0xc00091c300, {0x4b05a9?, 0xc000a6b0f0?}, 0xc0010bb0e0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:102 +0x30f fp=0xc001131098 sp=0xc001130fa0 pc=0x21a640f +github.com/rancher/opni/pkg/plugins/apis/system._System_UseManagementAPI_Handler.func1({0x5b426a8, 0xc0010bb1a0}, {0x4278de0?, 0xc0010bb0e0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:151 +0x83 fp=0xc0011310f0 sp=0xc001131098 pc=0x21ac063 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc0010bb1a0}, {0x4278de0, 0xc0010bb0e0}, 0xc0009979c0, 0xc000863d28) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc001131148 sp=0xc0011310f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc0010bb1a0}, {0x4278de0, 0xc0010bb0e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc0011311e8 sp=0xc001131148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc0010bb0b0}, {0x4278de0, 0xc0010bb0e0}, 0xc0009979c0, 0xc0010b6980) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc0011316b8 sp=0xc0011311e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc0010bb0b0}, {0x4278de0, 0xc0010bb0e0}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc001131740 sp=0xc0011316b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseManagementAPI_Handler({0x4278ea0?, 0xc00091c300}, {0x5b426a8, 0xc0010bb0b0}, 0xc0004b6e70, 0xc000b75440) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:153 +0x1c4 fp=0xc0011317d0 sp=0xc001131740 pc=0x21abec4 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc000f8b440, 0xc000c70f60, 0x771eca0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc001131db0 sp=0xc0011317d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc000f8b440, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc001131f28 sp=0xc001131db0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001131fe0 sp=0xc001131f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001131fe8 sp=0xc001131fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 247 [select]: +runtime.gopark(0xc001190fb0?, 0x2?, 0xa0?, 0xe?, 0xc001190f2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001190da8 sp=0xc001190d88 pc=0x4777ae +runtime.selectgo(0xc001190fb0, 0xc001190f28, 0x4b0571?, 0x0, 0x11847d3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001190ef8 sp=0xc001190da8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc001190fe0 sp=0xc001190ef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001190fe8 sp=0xc001190fe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 246 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 220 [chan receive]: +runtime.gopark(0x0?, 0x472ddb?, 0x71?, 0x5?, 0xc000bdcb08?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001112ef8 sp=0xc001112ed8 pc=0x4777ae +runtime.chanrecv(0xc000c9ccc0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001112f70 sp=0xc001112ef8 pc=0x441325 +runtime.chanrecv1(0xc000bdcaa0?, 0xc000e51440?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001112f98 sp=0xc001112f70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc001112fe0 sp=0xc001112f98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001112fe8 sp=0xc001112fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 218 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 221 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001207978 sp=0xc001207958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0012079b0 sp=0xc001207978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677cc0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0012079d0 sp=0xc0012079b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d16920, 0xc001207a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001207a18 sp=0xc0012079d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc000d16900) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001207b08 sp=0xc001207a18 pc=0x513f65 +net.(*netFD).accept(0xc000d16900) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001207c00 sp=0xc001207b08 pc=0x8967be +net.(*UnixListener).accept(0xc000d19800) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc001207c40 sp=0xc001207c00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc000d19800) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc001207cb0 sp=0xc001207c40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc000ee6de0) + :1 +0x44 fp=0xc001207ce8 sp=0xc001207cb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc0004ef680, {0x5b2c6a0, 0xc000ee6de0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc001207f18 sp=0xc001207ce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc001207f68 sp=0xc001207f18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc000ee71e0?, 0xc000fe06d0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001207fa8 sp=0xc001207f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001207fe0 sp=0xc001207fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001207fe8 sp=0xc001207fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 218 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 248 [select]: +runtime.gopark(0xc0011a3f90?, 0x2?, 0xb4?, 0x1b?, 0xc0011a3eac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011a3d28 sp=0xc0011a3d08 pc=0x4777ae +runtime.selectgo(0xc0011a3f90, 0xc0011a3ea8, 0x4b0571?, 0x0, 0x5d8cc5?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011a3e78 sp=0xc0011a3d28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc000001800) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc0011a3fc0 sp=0xc0011a3e78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc0011a3fe0 sp=0xc0011a3fc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011a3fe8 sp=0xc0011a3fe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 246 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 222 [select]: +runtime.gopark(0xc00118ef38?, 0x2?, 0xc0?, 0x19?, 0xc00118ef14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00118ed90 sp=0xc00118ed70 pc=0x4777ae +runtime.selectgo(0xc00118ef38, 0xc00118ef10, 0xc00118ef18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00118eee0 sp=0xc00118ed90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc00118ef68 sp=0xc00118eee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc000f1f2a8?, 0xc000fe06f0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc00118efa8 sp=0xc00118ef68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc00118efe0 sp=0xc00118efa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00118efe8 sp=0xc00118efe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 218 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 249 [select]: +runtime.gopark(0xc0011fff40?, 0x2?, 0xb4?, 0x1b?, 0xc0011ffecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001497d48 sp=0xc001497d28 pc=0x4777ae +runtime.selectgo(0xc001497f40, 0xc0011ffec8, 0xc000067078?, 0x0, 0xc000bdc6f8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001497e98 sp=0xc001497d48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc000831900}, 0xc000001890, 0x0, 0xc0011a41e0, 0xc000e9cec0) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc001497f70 sp=0xc001497e98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc001497fe0 sp=0xc001497f70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001497fe8 sp=0xc001497fe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 246 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 258 [chan receive]: +runtime.gopark(0x400a060?, 0xc00136dff0?, 0xe3?, 0x45?, 0x7?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00155bb28 sp=0xc00155bb08 pc=0x4777ae +runtime.chanrecv(0xc0011a4cc0, 0xc00120bcb8, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00155bba0 sp=0xc00155bb28 pc=0x441325 +runtime.chanrecv1(0xc0011c60d0?, 0x771d460?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00155bbc8 sp=0xc00155bba0 pc=0x440f72 +github.com/rancher/opni/pkg/gateway.(*StreamServer).OnPluginLoad.func1() + /home/ec2-user/opni/pkg/gateway/stream.go:285 +0x8eb fp=0xc00155bfe0 sp=0xc00155bbc8 pc=0x22bfbeb +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00155bfe8 sp=0xc00155bfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.(*StreamServer).OnPluginLoad in goroutine 209 + /home/ec2-user/opni/pkg/gateway/stream.go:255 +0xa52 + +goroutine 260 [select]: +runtime.gopark(0xc0010b1f90?, 0x2?, 0xff?, 0xff?, 0xc0010b1eac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010b1d28 sp=0xc0010b1d08 pc=0x4777ae +runtime.selectgo(0xc0010b1f90, 0xc0010b1ea8, 0x4b0571?, 0x0, 0xc000b85718?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010b1e78 sp=0xc0010b1d28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc000f24780) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc0010b1fc0 sp=0xc0010b1e78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc0010b1fe0 sp=0xc0010b1fc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010b1fe8 sp=0xc0010b1fe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 258 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 223 [chan receive]: +runtime.gopark(0x22c2125?, 0xc000b80f50?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00118fef8 sp=0xc00118fed8 pc=0x4777ae +runtime.chanrecv(0xc000c9ccc0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00118ff70 sp=0xc00118fef8 pc=0x441325 +runtime.chanrecv1(0xc000bdcaa0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00118ff98 sp=0xc00118ff70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc00118ffe0 sp=0xc00118ff98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00118ffe8 sp=0xc00118ffe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 216 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 261 [select]: +runtime.gopark(0xc0011e9f40?, 0x2?, 0xb4?, 0x1b?, 0xc0011e9ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00155dd48 sp=0xc00155dd28 pc=0x4777ae +runtime.selectgo(0xc00155df40, 0xc0011e9ec8, 0xc00110a030?, 0x0, 0x448855?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00155de98 sp=0xc00155dd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc0009d76d0}, 0xc000f24810, 0x0, 0xc0010bc6c0, 0xc0010b4af0) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc00155df70 sp=0xc00155de98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc00155dfe0 sp=0xc00155df70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00155dfe8 sp=0xc00155dfe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 258 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 224 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001217978 sp=0xc001217958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0012179b0 sp=0xc001217978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677db8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0012179d0 sp=0xc0012179b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d167a0, 0xc001217a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001217a18 sp=0xc0012179d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc000d16780) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001217b08 sp=0xc001217a18 pc=0x513f65 +net.(*netFD).accept(0xc000d16780) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001217c00 sp=0xc001217b08 pc=0x8967be +net.(*UnixListener).accept(0xc000d197a0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc001217c40 sp=0xc001217c00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc000d197a0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc001217cb0 sp=0xc001217c40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc000ee6d40) + :1 +0x44 fp=0xc001217ce8 sp=0xc001217cb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc0004ef860, {0x5b2c6a0, 0xc000ee6d40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc001217f18 sp=0xc001217ce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc001217f68 sp=0xc001217f18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc000ee74a0?, 0xc000fe0940?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001217fa8 sp=0xc001217f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001217fe0 sp=0xc001217fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001217fe8 sp=0xc001217fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 216 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 225 [select]: +runtime.gopark(0xc000ef7f38?, 0x2?, 0x2?, 0x0?, 0xc000ef7f14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000ef7d90 sp=0xc000ef7d70 pc=0x4777ae +runtime.selectgo(0xc000ef7f38, 0xc000ef7f10, 0xc000ef7f18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000ef7ee0 sp=0xc000ef7d90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc000ef7f68 sp=0xc000ef7ee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc000f1f440?, 0xc000fe0960?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc000ef7fa8 sp=0xc000ef7f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc000ef7fe0 sp=0xc000ef7fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000ef7fe8 sp=0xc000ef7fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 216 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 240 [select]: +runtime.gopark(0xc001201f70?, 0x2?, 0xb4?, 0x1b?, 0xc001201ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001201d20 sp=0xc001201d00 pc=0x4777ae +runtime.selectgo(0xc001201f70, 0xc001201eb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001201e70 sp=0xc001201d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0009d2160, {0x5b426e0, 0xc00010f860}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001201fa0 sp=0xc001201e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001201fe0 sp=0xc001201fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001201fe8 sp=0xc001201fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 237 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 424 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x80?, 0xde?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001557cd0 sp=0xc001557cb0 pc=0x4777ae +runtime.chanrecv(0xc00127bc80, 0xc001557de0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001557d48 sp=0xc001557cd0 pc=0x441325 +runtime.chanrecv1(0xc001557df0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001557d70 sp=0xc001557d48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc001557f30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc001557e00 sp=0xc001557d70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc000ee4410, 0x420c60?, 0xc0015de210) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc001557fa0 sp=0xc001557e00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc001557fe0 sp=0xc001557fa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001557fe8 sp=0xc001557fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 420 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 250 [chan receive]: +runtime.gopark(0xc0008313b0?, 0x4b05a9?, 0x50?, 0xce?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011dce20 sp=0xc0011dce00 pc=0x4777ae +runtime.chanrecv(0xc00119a240, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0011dce98 sp=0xc0011dce20 pc=0x441325 +runtime.chanrecv1(0xc000eb99b0?, 0x48a0bb?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0011dcec0 sp=0xc0011dce98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc0011dcfe0 sp=0xc0011dcec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011dcfe8 sp=0xc0011dcfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 246 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 251 [chan receive]: +runtime.gopark(0xc0008313b0?, 0x4b05a9?, 0x50?, 0xde?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011dde20 sp=0xc0011dde00 pc=0x4777ae +runtime.chanrecv(0xc00119a240, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0011dde98 sp=0xc0011dde20 pc=0x441325 +runtime.chanrecv1(0xc000eb99b0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0011ddec0 sp=0xc0011dde98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc0011ddfe0 sp=0xc0011ddec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011ddfe8 sp=0xc0011ddfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 246 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 252 [select]: +runtime.gopark(0xc001183aa8?, 0x2?, 0x67?, 0x6a?, 0xc0011839c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001183838 sp=0xc001183818 pc=0x4777ae +runtime.selectgo(0xc001183aa8, 0xc0011839c0, 0xc0011839d0?, 0x0, 0x43f1c20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001183988 sp=0xc001183838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc000be0aa0) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc001183ad8 sp=0xc001183988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc001196480, {0x5b426a8, 0xc000eb99b0}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc001183cf8 sp=0xc001183ad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc001183fe0 sp=0xc001183cf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001183fe8 sp=0xc001183fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 246 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 253 [chan receive]: +runtime.gopark(0x70?, 0x0?, 0x70?, 0x0?, 0xc000c9df80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011dee20 sp=0xc0011dee00 pc=0x4777ae +runtime.chanrecv(0xc000c9df80, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0011dee98 sp=0xc0011dee20 pc=0x441325 +runtime.chanrecv1(0xc000eb9470?, 0x44063c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0011deec0 sp=0xc0011dee98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc0011defe0 sp=0xc0011deec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011defe8 sp=0xc0011defe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 258 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 254 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc000c9df80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011dfe20 sp=0xc0011dfe00 pc=0x4777ae +runtime.chanrecv(0xc000c9df80, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0011dfe98 sp=0xc0011dfe20 pc=0x441325 +runtime.chanrecv1(0xc000eb9470?, 0x4aead9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0011dfec0 sp=0xc0011dfe98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc0011dffe0 sp=0xc0011dfec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011dffe8 sp=0xc0011dffe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 258 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 285 [select]: +runtime.gopark(0xc001433e70?, 0x2?, 0x0?, 0x0?, 0xc001433df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001de1c58 sp=0xc001de1c38 pc=0x4777ae +runtime.selectgo(0xc001de1e70, 0xc001433df0, 0xc001433e00?, 0x0, 0xc0010b8a80?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001de1da8 sp=0xc001de1c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00139c0a0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001de1ea0 sp=0xc001de1da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000572d90) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001de1f58 sp=0xc001de1ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc001de1fe0 sp=0xc001de1f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001de1fe8 sp=0xc001de1fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 322 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 289 [select]: +runtime.gopark(0xc002607e48?, 0x2?, 0x0?, 0x0?, 0xc002607dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002607c30 sp=0xc002607c10 pc=0x4777ae +runtime.selectgo(0xc002607e48, 0xc002607dc8, 0xc002607dd8?, 0x0, 0xc000f89c00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002607d80 sp=0xc002607c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00139c280, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002607e78 sp=0xc002607d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0005730a0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002607f30 sp=0xc002607e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc002607fe0 sp=0xc002607f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002607fe8 sp=0xc002607fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 303 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 278 [select]: +runtime.gopark(0xc00124c890?, 0x2?, 0x98?, 0xc7?, 0xc00124c83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00124c6a8 sp=0xc00124c688 pc=0x4777ae +runtime.selectgo(0xc00124c890, 0xc00124c838, 0x7f429a422b00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00124c7f8 sp=0xc00124c6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00010ff40, {0xc001150460, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc00124c8c0 sp=0xc00124c7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00010ff40, {0xc001150460, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc00124c958 sp=0xc00124c8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001150120, {0xc001150460, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc00124c9a0 sp=0xc00124c958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001150120}, {0xc001150460, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00124c9e8 sp=0xc00124c9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00110ed80, {0xc001150460, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc00124ca58 sp=0xc00124c9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001150450, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc00124cb28 sp=0xc00124ca58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001160180?, 0xc00110ed80, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc00124cc18 sp=0xc00124cb28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001150450, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001152500}, 0xc000c70240?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc00124ccd0 sp=0xc00124cc18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000c074a0, {0x43dcf60?, 0xc001152500}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc00124cf08 sp=0xc00124ccd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0011505a0, {0x43dcf60, 0xc001152500}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc00124cff0 sp=0xc00124cf08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0009d25c0, {0x43dcf60, 0xc001152500}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc00124d028 sp=0xc00124cff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00114c520) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc00124d068 sp=0xc00124d028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00114c520?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc00124d0a8 sp=0xc00124d068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00114c530) + :1 +0x46 fp=0xc00124d0e0 sp=0xc00124d0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c703c0, {0x5b51368, 0xc00114c530}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc00124d320 sp=0xc00124d0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000c6e5e0, {0x5b51038, 0xc00114c520}) + :1 +0xaf fp=0xc00124d370 sp=0xc00124d320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000c6e5e0}, {0x5b4a6f0?, 0xc0009d25c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc00124d3b0 sp=0xc00124d370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc0011505a0}, 0xc000dc3020?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc00124d400 sp=0xc00124d3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc0011505a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc00124d490 sp=0xc00124d400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc000c074a0}, 0xc000dc3020, 0xc00112b580) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc00124d7d0 sp=0xc00124d490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc000c074a0}, 0x3f252a0?, 0xc00114c450?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc00124d848 sp=0xc00124d7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00110ed80, 0xc000c70450, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc00124ddb0 sp=0xc00124d848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00110ed80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc00124df28 sp=0xc00124ddb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc00124dfe0 sp=0xc00124df28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00124dfe8 sp=0xc00124dfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 264 [select]: +runtime.gopark(0xc00119ffb0?, 0x2?, 0xb4?, 0x1b?, 0xc00119ff74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00119fde8 sp=0xc00119fdc8 pc=0x4777ae +runtime.selectgo(0xc00119ffb0, 0xc00119ff70, 0x0?, 0x0, 0xc000000000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00119ff38 sp=0xc00119fde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc00119ffe0 sp=0xc00119ff38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00119ffe8 sp=0xc00119ffe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 265 [select]: +runtime.gopark(0xc0011d8fb0?, 0x2?, 0xb0?, 0x8d?, 0xc0011d8ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011d8d58 sp=0xc0011d8d38 pc=0x4777ae +runtime.selectgo(0xc0011d8fb0, 0xc0011d8ee0, 0x4aead9?, 0x0, 0xc000f1df30?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011d8ea8 sp=0xc0011d8d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0011d8fe0 sp=0xc0011d8ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011d8fe8 sp=0xc0011d8fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 266 [chan receive]: +runtime.gopark(0x900000000000000?, 0x0?, 0x4?, 0x0?, 0x1?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001192d60 sp=0xc001192d40 pc=0x4777ae +runtime.chanrecv(0xc000f26cc0, 0xc001192e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001192dd8 sp=0xc001192d60 pc=0x441325 +runtime.chanrecv1(0xd?, 0x14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001192e00 sp=0xc001192dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001192fe0 sp=0xc001192e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001192fe8 sp=0xc001192fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 255 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc000c9df80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001240e20 sp=0xc001240e00 pc=0x4777ae +runtime.chanrecv(0xc000c9df80, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001240e98 sp=0xc001240e20 pc=0x441325 +runtime.chanrecv1(0xc000eb9470?, 0x2106cfa?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001240ec0 sp=0xc001240e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc001240fe0 sp=0xc001240ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001240fe8 sp=0xc001240fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 258 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 256 [select]: +runtime.gopark(0xc000a6daa8?, 0x2?, 0x67?, 0x6a?, 0xc000a6d9c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a6d838 sp=0xc000a6d818 pc=0x4777ae +runtime.selectgo(0xc000a6daa8, 0xc000a6d9c0, 0xc000a6d9d0?, 0x0, 0x43f1c20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a6d988 sp=0xc000a6d838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc00098d860) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc000a6dad8 sp=0xc000a6d988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc000f8b560, {0x5b426a8, 0xc000eb9470}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc000a6dcf8 sp=0xc000a6dad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc000a6dfe0 sp=0xc000a6dcf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a6dfe8 sp=0xc000a6dfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 258 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 348 [select]: +runtime.gopark(0xc00129a890?, 0x2?, 0x98?, 0xa7?, 0xc00129a83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00129a6a8 sp=0xc00129a688 pc=0x4777ae +runtime.selectgo(0xc00129a890, 0xc00129a838, 0x7f429a426a00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00129a7f8 sp=0xc00129a6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0007c1a90, {0xc001266b50, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc00129a8c0 sp=0xc00129a7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0007c1a90, {0xc001266b50, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc00129a958 sp=0xc00129a8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001266ab0, {0xc001266b50, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc00129a9a0 sp=0xc00129a958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001266ab0}, {0xc001266b50, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00129a9e8 sp=0xc00129a9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00123cc60, {0xc001266b50, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc00129aa58 sp=0xc00129a9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001266b40, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc00129ab28 sp=0xc00129aa58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc00141c140?, 0xc00123cc60, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc00129ac18 sp=0xc00129ab28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001266b40, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0007c1db0}, 0xc000c70240?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc00129acd0 sp=0xc00129ac18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc0012340f0, {0x43dcf60?, 0xc0007c1db0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc00129af08 sp=0xc00129acd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001266c90, {0x43dcf60, 0xc0007c1db0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc00129aff0 sp=0xc00129af08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc00127c140, {0x43dcf60, 0xc0007c1db0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc00129b028 sp=0xc00129aff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc000fe1a60) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc00129b068 sp=0xc00129b028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc000fe1a60?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc00129b0a8 sp=0xc00129b068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc000fe1a70) + :1 +0x46 fp=0xc00129b0e0 sp=0xc00129b0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c703c0, {0x5b51368, 0xc000fe1a70}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc00129b320 sp=0xc00129b0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000c6e5e0, {0x5b51038, 0xc000fe1a60}) + :1 +0xaf fp=0xc00129b370 sp=0xc00129b320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000c6e5e0}, {0x5b4a6f0?, 0xc00127c140}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc00129b3b0 sp=0xc00129b370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc001266c90}, 0xc000f1fd88?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc00129b400 sp=0xc00129b3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc001266c90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc00129b490 sp=0xc00129b400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc0012340f0}, 0xc000f1fd88, 0xc001237540) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc00129b7d0 sp=0xc00129b490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc0012340f0}, 0x3f252a0?, 0xc000fe1990?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc00129b848 sp=0xc00129b7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00123cc60, 0xc000c70450, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc00129bdb0 sp=0xc00129b848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00123cc60, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc00129bf28 sp=0xc00129bdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc00129bfe0 sp=0xc00129bf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00129bfe8 sp=0xc00129bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 287 [select]: +runtime.gopark(0xc00115cf70?, 0x4?, 0x1d?, 0x0?, 0xc00115cdd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00115cc00 sp=0xc00115cbe0 pc=0x4777ae +runtime.selectgo(0xc00115cf70, 0xc00115cdd0, 0x1?, 0x0, 0x5b5d9d8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00115cd50 sp=0xc00115cc00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001174340) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc00115cfc0 sp=0xc00115cd50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc00115cfe0 sp=0xc00115cfc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00115cfe8 sp=0xc00115cfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 304 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 342 [IO wait]: +runtime.gopark(0xc00113efe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00113ef98 sp=0xc00113ef78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00113efd0 sp=0xc00113ef98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42416779d8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00113eff0 sp=0xc00113efd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0010e00a0, 0xc00108a000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00113f038 sp=0xc00113eff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0010e0080, {0xc00108a000, 0x4000, 0x4000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00113f128 sp=0xc00113f038 pc=0x50e5a5 +net.(*netFD).Read(0xc0010e0080, {0xc00108a000, 0x4000, 0x4000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00113f188 sp=0xc00113f128 pc=0x89342b +net.(*conn).Read(0xc00007de18, {0xc00108a000, 0x4000, 0x4000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00113f210 sp=0xc00113f188 pc=0x8b176d +net.(*UnixConn).Read(0xc00007de18, {0xc00108a000, 0x4000, 0x4000}) + :1 +0x54 fp=0xc00113f258 sp=0xc00113f210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0010d9590, {0xc00108a000, 0x4000, 0x4000}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc00113f2b8 sp=0xc00113f258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0010b89a8, {0x5b0f6c0, 0xc0010d9590}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc00113f320 sp=0xc00113f2b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc0010b8700, {0x5b0f840?, 0xc00007de18}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc00113f390 sp=0xc00113f320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc0010b8700, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc00113f9c0 sp=0xc00113f390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc0010b8700, {0xc001400000, 0x8000, 0xc001ed4410?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc00113faa8 sp=0xc00113f9c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc000ee7fa0, {0xc001400000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc00113faf0 sp=0xc00113faa8 pc=0x10969ed +bufio.(*Reader).Read(0xc00127a3c0, {0xc000f8e3c0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00113fbb0 sp=0xc00113faf0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00127a3c0}, {0xc000f8e3c0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00113fbf8 sp=0xc00113fbb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000f8e3c0, 0x9, 0x9}, {0x5b0f0a0, 0xc00127a3c0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00113fc98 sp=0xc00113fbf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000f8e380) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00113fdd0 sp=0xc00113fc98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000f95440, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc00113ffb0 sp=0xc00113fdd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc00113ffe0 sp=0xc00113ffb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00113ffe8 sp=0xc00113ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 323 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 512 [select]: +runtime.gopark(0xc000a4ffb0?, 0x2?, 0xb4?, 0x1b?, 0xc000a4ff74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a4fde8 sp=0xc000a4fdc8 pc=0x4777ae +runtime.selectgo(0xc000a4ffb0, 0xc000a4ff70, 0x1000000?, 0x0, 0x116e9f5?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a4ff38 sp=0xc000a4fde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc000a4ffe0 sp=0xc000a4ff38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a4ffe8 sp=0xc000a4ffe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 566 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 257 [select]: +runtime.gopark(0xc0011813f8?, 0x2?, 0xc0?, 0x72?, 0xc00118137c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011811e8 sp=0xc0011811c8 pc=0x4777ae +runtime.selectgo(0xc0011813f8, 0xc001181378, 0x5b203e8?, 0x0, 0x2710?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001181338 sp=0xc0011811e8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000830e60, {0xc000eb9540, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc001181428 sp=0xc001181338 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000830e60, {0xc000eb9540, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc0011814c0 sp=0xc001181428 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000eb9500, {0xc000eb9540, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001181508 sp=0xc0011814c0 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc000eb9500}, {0xc000eb9540, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001181550 sp=0xc001181508 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001196120, {0xc000eb9540, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0011815c0 sp=0xc001181550 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc000eb9530, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001181690 sp=0xc0011815c0 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc001196120, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001181780 sp=0xc001181690 pc=0x1171c5c +google.golang.org/grpc.recv(0xc000eb9530, {0x7f424347ae80, 0x779c980}, 0x7f429a503290?, {0x0, 0x0}, {0x43f1c20, 0xc0005ba0e0}, 0x456976?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001181838 sp=0xc001181780 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc000eb7110, {0x43f1c20?, 0xc0005ba0e0}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc001181a98 sp=0xc001181838 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00094ff14?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001181af8 sp=0xc001181a98 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00094fe60, 0xc001181cb0, 0xc001181c80) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc001181ba0 sp=0xc001181af8 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00094fe60, {0x43f1c20, 0xc0005ba0e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001181ce0 sp=0xc001181ba0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc000ed13c0, {0x43f1c20, 0xc0005ba0e0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc001181ea8 sp=0xc001181ce0 pc=0x12ac5b9 +github.com/rancher/opni/pkg/plugins/apis/apiextensions.(*streamAPIExtensionConnectInternalClient).Recv(0xc000e9cbf0) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/apiextensions_grpc.pb.go:249 +0x66 fp=0xc001181ee8 sp=0xc001181ea8 pc=0x218dd66 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc001181fe0 sp=0xc001181ee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001181fe8 sp=0xc001181fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 256 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 372 [select]: +runtime.gopark(0xc00125c890?, 0x2?, 0x4e?, 0x1a?, 0xc00125c83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00125c6a8 sp=0xc00125c688 pc=0x4777ae +runtime.selectgo(0xc00125c890, 0xc00125c838, 0x7f429a428b00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00125c7f8 sp=0xc00125c6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00139c4b0, {0xc001394fa0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc00125c8c0 sp=0xc00125c7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00139c4b0, {0xc001394fa0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc00125c958 sp=0xc00125c8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001394d50, {0xc001394fa0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc00125c9a0 sp=0xc00125c958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001394d50}, {0xc001394fa0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00125c9e8 sp=0xc00125c9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00110f440, {0xc001394fa0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc00125ca58 sp=0xc00125c9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001394f90, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc00125cb28 sp=0xc00125ca58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0014802c0?, 0xc00110f440, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc00125cc18 sp=0xc00125cb28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001394f90, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc00139c8c0}, 0xc000c4a360?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc00125ccd0 sp=0xc00125cc18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000c07680, {0x43dcf60?, 0xc00139c8c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc00125cf08 sp=0xc00125ccd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0013950e0, {0x43dcf60, 0xc00139c8c0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc00125cff0 sp=0xc00125cf08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0009d3e00, {0x43dcf60, 0xc00139c8c0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc00125d028 sp=0xc00125cff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00114dd40) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc00125d068 sp=0xc00125d028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00114dd40?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc00125d0a8 sp=0xc00125d068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00114dd50) + :1 +0x46 fp=0xc00125d0e0 sp=0xc00125d0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c4a4e0, {0x5b51368, 0xc00114dd50}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc00125d320 sp=0xc00125d0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000bfd440, {0x5b51038, 0xc00114dd40}) + :1 +0xaf fp=0xc00125d370 sp=0xc00125d320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000bfd440}, {0x5b4a6f0?, 0xc0009d3e00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc00125d3b0 sp=0xc00125d370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc0013950e0}, 0xc000dc3c08?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc00125d400 sp=0xc00125d3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc0013950e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc00125d490 sp=0xc00125d400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc000c07680}, 0xc000dc3c08, 0xc0011618c0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc00125d7d0 sp=0xc00125d490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc000c07680}, 0x3f252a0?, 0xc00114dc70?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc00125d848 sp=0xc00125d7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc00110f440, 0xc000c4a570, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc00125ddb0 sp=0xc00125d848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc00110f440, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc00125df28 sp=0xc00125ddb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc00125dfe0 sp=0xc00125df28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00125dfe8 sp=0xc00125dfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 291 [select]: +runtime.gopark(0xc001099fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001099f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001099de8 sp=0xc001099dc8 pc=0x4777ae +runtime.selectgo(0xc001099fb0, 0xc001099f70, 0x0?, 0x0, 0xc000000000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001099f38 sp=0xc001099de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001099fe0 sp=0xc001099f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001099fe8 sp=0xc001099fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 292 [select]: +runtime.gopark(0xc001245fb0?, 0x2?, 0xb0?, 0x5d?, 0xc001245ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001245d58 sp=0xc001245d38 pc=0x4777ae +runtime.selectgo(0xc001245fb0, 0xc001245ee0, 0x0?, 0x0, 0x448845?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001245ea8 sp=0xc001245d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001245fe0 sp=0xc001245ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001245fe8 sp=0xc001245fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 293 [chan receive]: +runtime.gopark(0x7700000000000000?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001191d60 sp=0xc001191d40 pc=0x4777ae +runtime.chanrecv(0xc00123e120, 0xc001191e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001191dd8 sp=0xc001191d60 pc=0x441325 +runtime.chanrecv1(0x28?, 0x14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001191e00 sp=0xc001191dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001191fe0 sp=0xc001191e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001191fe8 sp=0xc001191fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 306 [select]: +runtime.gopark(0xc00113b710?, 0x2?, 0x0?, 0x0?, 0xc00113b6bc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00113b528 sp=0xc00113b508 pc=0x4777ae +runtime.selectgo(0xc00113b710, 0xc00113b6b8, 0x7f429a41c200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00113b678 sp=0xc00113b528 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000831400, {0xc000eb98a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc00113b740 sp=0xc00113b678 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000831400, {0xc000eb98a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc00113b7d8 sp=0xc00113b740 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000eb9800, {0xc000eb98a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc00113b820 sp=0xc00113b7d8 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc000eb9800}, {0xc000eb98a0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00113b868 sp=0xc00113b820 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001196360, {0xc000eb98a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc00113b8d8 sp=0xc00113b868 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc000eb9890, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc00113b9a8 sp=0xc00113b8d8 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc001196360, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc00113ba98 sp=0xc00113b9a8 pc=0x1171c5c +google.golang.org/grpc.recv(0xc000eb9890, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43f1c20, 0xc000905b90}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc00113bb50 sp=0xc00113ba98 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000e842d0, {0x43f1c20?, 0xc000905b90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc00113bd88 sp=0xc00113bb50 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc000eb99e0, {0x43f1c20, 0xc000905b90}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc00113be70 sp=0xc00113bd88 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0005fb7c0, {0x43f1c20, 0xc000905b90}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc00113bea8 sp=0xc00113be70 pc=0x1fd94b2 +github.com/rancher/opni/pkg/plugins/apis/apiextensions.(*streamAPIExtensionConnectInternalServer).Recv(0xc000e9ce00) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/apiextensions_grpc.pb.go:303 +0x66 fp=0xc00113bee8 sp=0xc00113bea8 pc=0x218e046 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc00113bfe0 sp=0xc00113bee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00113bfe8 sp=0xc00113bfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 252 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 294 [select]: +runtime.gopark(0xc001246f20?, 0x2?, 0xb4?, 0x1b?, 0xc001246eb4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001246d28 sp=0xc001246d08 pc=0x4777ae +runtime.selectgo(0xc001246f20, 0xc001246eb0, 0x0?, 0x0, 0xc001246f18?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001246e78 sp=0xc001246d28 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc00110a780, 0xc000fe0ed0, 0xc000fe0ee0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001246f50 sp=0xc001246e78 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeKeyValueStore(0xc00110a780, {0x5b43370?, 0xc000fe0dd0}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:214 +0x1a8 fp=0xc001246f98 sp=0xc001246f50 pc=0x21a84a8 +github.com/rancher/opni/pkg/gateway.NewGateway.func2.1() + /home/ec2-user/opni/pkg/gateway/gateway.go:157 +0x62 fp=0xc001246fe0 sp=0xc001246f98 pc=0x22b18c2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001246fe8 sp=0xc001246fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.NewGateway.func2 in goroutine 233 + /home/ec2-user/opni/pkg/gateway/gateway.go:157 +0x3cc + +goroutine 270 [select]: +runtime.gopark(0xc00109df70?, 0x2?, 0xb4?, 0x1b?, 0xc00109debc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00109dd20 sp=0xc00109dd00 pc=0x4777ae +runtime.selectgo(0xc00109df70, 0xc00109deb8, 0x4aead9?, 0x0, 0x215774d?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00109de70 sp=0xc00109dd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00006d580, {0x5b426e0, 0xc000b68b40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc00109dfa0 sp=0xc00109de70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc00109dfe0 sp=0xc00109dfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00109dfe8 sp=0xc00109dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 262 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 271 [select]: +runtime.gopark(0xc001213f70?, 0x2?, 0xb4?, 0x1b?, 0xc001213ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001213d20 sp=0xc001213d00 pc=0x4777ae +runtime.selectgo(0xc001213f70, 0xc001213eb8, 0x472ddb?, 0x0, 0x4ad421?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001213e70 sp=0xc001213d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00006d6e0, {0x5b426e0, 0xc000b68b90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001213fa0 sp=0xc001213e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001213fe0 sp=0xc001213fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001213fe8 sp=0xc001213fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 262 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 297 [select]: +runtime.gopark(0xc001251448?, 0x2?, 0xd0?, 0x12?, 0xc0012513fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001251278 sp=0xc001251258 pc=0x4777ae +runtime.selectgo(0xc001251448, 0xc0012513f8, 0xc000b6efe8?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0012513c8 sp=0xc001251278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00123c7e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001251478 sp=0xc0012513c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00123a410, {0x42219c0?, 0xc000ff9620}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0012516d8 sp=0xc001251478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00123c654?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001251738 sp=0xc0012516d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00123c5a0, 0xc0012518f0, 0xc0012518c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc0012517e0 sp=0xc001251738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00123c5a0, {0x42219c0, 0xc000ff9620}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001251920 sp=0xc0012517e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc000ff96e0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc000ff95f0}, {0x42219c0, 0xc000ff9620}, 0x0?, {0xc0005fac80, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc0012519c0 sp=0xc001251920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc000bdcaa0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc000ff95f0}, {0x42219c0, 0xc000ff9620}, 0xc0009c7400, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc001251d30 sp=0xc0012519c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc0009c7400, {0x5b426e0, 0xc000bdcaa0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc000ff95f0}, {0x42219c0, 0xc000ff9620}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc001251e58 sp=0xc001251d30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseKeyValueStore(0xc000c6f160, {0x5b426e0, 0xc000bdcaa0}, 0x7769700?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:69 +0xee fp=0xc001251ee8 sp=0xc001251e58 pc=0x21ab96e +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeKeyValueStore.func2(0x3) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:219 +0xf2 fp=0xc001251f58 sp=0xc001251ee8 pc=0x21a85d2 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc001251fe0 sp=0xc001251f58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001251fe8 sp=0xc001251fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 294 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 272 [select]: +runtime.gopark(0xc001215f70?, 0x2?, 0xb4?, 0x1b?, 0xc001215ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001215d20 sp=0xc001215d00 pc=0x4777ae +runtime.selectgo(0xc001215f70, 0xc001215eb8, 0x472ddb?, 0x0, 0x4ad421?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001215e70 sp=0xc001215d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00006dac0, {0x5b426e0, 0xc000b68d70}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001215fa0 sp=0xc001215e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001215fe0 sp=0xc001215fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001215fe8 sp=0xc001215fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 177 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 273 [select]: +runtime.gopark(0xc000a6bf70?, 0x2?, 0xb4?, 0x1b?, 0xc000a6bebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a6bd20 sp=0xc000a6bd00 pc=0x4777ae +runtime.selectgo(0xc000a6bf70, 0xc000a6beb8, 0x4b3745?, 0x0, 0xc000bdcb08?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a6be70 sp=0xc000a6bd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b30040, {0x5b426e0, 0xc000b68dc0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc000a6bfa0 sp=0xc000a6be70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc000a6bfe0 sp=0xc000a6bfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a6bfe8 sp=0xc000a6bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 177 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 298 [chan receive]: +runtime.gopark(0x78?, 0x7769700?, 0x70?, 0x67?, 0xc000bdcb08?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001158ef8 sp=0xc001158ed8 pc=0x4777ae +runtime.chanrecv(0xc000c9ccc0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001158f70 sp=0xc001158ef8 pc=0x441325 +runtime.chanrecv1(0xc000bdcaa0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001158f98 sp=0xc001158f70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc001158fe0 sp=0xc001158f98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001158fe8 sp=0xc001158fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 296 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 299 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001219978 sp=0xc001219958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0012199b0 sp=0xc001219978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677ad0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0012199d0 sp=0xc0012199b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d175a0, 0xc001219a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001219a18 sp=0xc0012199d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc000d17580) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001219b08 sp=0xc001219a18 pc=0x513f65 +net.(*netFD).accept(0xc000d17580) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001219c00 sp=0xc001219b08 pc=0x8967be +net.(*UnixListener).accept(0xc000ff98c0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc001219c40 sp=0xc001219c00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc000ff98c0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc001219cb0 sp=0xc001219c40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc000ee7a00) + :1 +0x44 fp=0xc001219ce8 sp=0xc001219cb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc0004efe00, {0x5b2c6a0, 0xc000ee7a00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc001219f18 sp=0xc001219ce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc001219f68 sp=0xc001219f18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc000ee7b60?, 0xc000fe10e0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001219fa8 sp=0xc001219f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001219fe0 sp=0xc001219fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001219fe8 sp=0xc001219fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 296 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 300 [select]: +runtime.gopark(0xc001247f38?, 0x2?, 0x0?, 0x0?, 0xc001247f14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001247d90 sp=0xc001247d70 pc=0x4777ae +runtime.selectgo(0xc001247f38, 0xc001247f10, 0xc001247f18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001247ee0 sp=0xc001247d90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc001247f68 sp=0xc001247ee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc000f1fa28?, 0xc000fe1100?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001247fa8 sp=0xc001247f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001247fe0 sp=0xc001247fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001247fe8 sp=0xc001247fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 296 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 307 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x10?, 0x8e?, 0x10?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001258cf8 sp=0xc001258cd8 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001258d70 sp=0xc001258cf8 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0x5b55580?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001258d98 sp=0xc001258d70 pc=0x440f72 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseKeyValueStore(0xc0005a8c80, {0x5b43f90?, 0xc001490050}) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:185 +0xc25 fp=0xc001258fd0 sp=0xc001258d98 pc=0x3970905 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseKeyValueStore(0xc00091c300, {0x4b05a9?, 0xc00113d0f0?}, 0xc0011ec720) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:123 +0x18d fp=0xc001259098 sp=0xc001258fd0 pc=0x21a6c2d +github.com/rancher/opni/pkg/plugins/apis/system._System_UseKeyValueStore_Handler.func1({0x5b426a8, 0xc0011ec7e0}, {0x4278de0?, 0xc0011ec720}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:187 +0x83 fp=0xc0012590f0 sp=0xc001259098 pc=0x21ac823 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc0011ec7e0}, {0x4278de0, 0xc0011ec720}, 0xc00043b060, 0xc000a736b0) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc001259148 sp=0xc0012590f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc0011ec7e0}, {0x4278de0, 0xc0011ec720}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc0012591e8 sp=0xc001259148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc0011ec6f0}, {0x4278de0, 0xc0011ec720}, 0xc00043b060, 0xc0011ca740) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc0012596b8 sp=0xc0012591e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc0011ec6f0}, {0x4278de0, 0xc0011ec720}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc001259740 sp=0xc0012596b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseKeyValueStore_Handler({0x4278ea0?, 0xc00091c300}, {0x5b426a8, 0xc0011ec6f0}, 0xc000905810, 0xc000b75440) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:189 +0x1c4 fp=0xc0012597d0 sp=0xc001259740 pc=0x21ac684 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc001197200, 0xc000c70f60, 0x771ecd0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc001259db0 sp=0xc0012597d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc001197200, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc001259f28 sp=0xc001259db0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001259fe0 sp=0xc001259f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001259fe8 sp=0xc001259fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1293 [select]: +runtime.gopark(0xc001ef6f48?, 0x2?, 0x10?, 0x98?, 0xc001ef6f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ef6d80 sp=0xc001ef6d60 pc=0x4777ae +runtime.selectgo(0xc001ef6f48, 0xc001ef6f00, 0x1?, 0x0, 0x86?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ef6ed0 sp=0xc001ef6d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).Recv(0xc001ff19c0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:234 +0xeb fp=0xc001ef6f78 sp=0xc001ef6ed0 pc=0x2155d2b +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc0026a4a00) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc001ef6fc0 sp=0xc001ef6f78 pc=0x215759f +github.com/hashicorp/go-plugin.newGRPCClient.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x34 fp=0xc001ef6fe0 sp=0xc001ef6fc0 pc=0x21599b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ef6fe8 sp=0xc001ef6fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 1361 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x5b7 + +goroutine 457 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0xa0?, 0x2f?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00157fcd0 sp=0xc00157fcb0 pc=0x4777ae +runtime.chanrecv(0xc000f1acc0, 0xc00157fde0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00157fd48 sp=0xc00157fcd0 pc=0x441325 +runtime.chanrecv1(0xc00157fdf0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00157fd70 sp=0xc00157fd48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc00157ff30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc00157fe00 sp=0xc00157fd70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc000ee4410, 0x420c60?, 0xc000ff8a20) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc00157ffa0 sp=0xc00157fe00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc00157ffe0 sp=0xc00157ffa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00157ffe8 sp=0xc00157ffe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 456 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 308 [select]: +runtime.gopark(0xc0010f2890?, 0x2?, 0x98?, 0x27?, 0xc0010f283c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010f26a8 sp=0xc0010f2688 pc=0x4777ae +runtime.selectgo(0xc0010f2890, 0xc0010f2838, 0x7f429a420700?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010f27f8 sp=0xc0010f26a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0011f2190, {0xc0011ecc40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0010f28c0 sp=0xc0010f27f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0011f2190, {0xc0011ecc40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0010f2958 sp=0xc0010f28c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0011ec9c0, {0xc0011ecc40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0010f29a0 sp=0xc0010f2958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0011ec9c0}, {0xc0011ecc40, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0010f29e8 sp=0xc0010f29a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001197320, {0xc0011ecc40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0010f2a58 sp=0xc0010f29e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0011ecc30, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0010f2b28 sp=0xc0010f2a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0011cb380?, 0xc001197320, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0010f2c18 sp=0xc0010f2b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0011ecc30, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0011f25a0}, 0xc000c4a360?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0010f2cd0 sp=0xc0010f2c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000e84f00, {0x43dcf60?, 0xc0011f25a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0010f2f08 sp=0xc0010f2cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0011ecd80, {0x43dcf60, 0xc0011f25a0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0010f2ff0 sp=0xc0010f2f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc00043b340, {0x43dcf60, 0xc0011f25a0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0010f3028 sp=0xc0010f2ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc0011cec70) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc0010f3068 sp=0xc0010f3028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc0011cec70?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0010f30a8 sp=0xc0010f3068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc0011cec80) + :1 +0x46 fp=0xc0010f30e0 sp=0xc0010f30a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c4a4e0, {0x5b51368, 0xc0011cec80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc0010f3320 sp=0xc0010f30e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000bfd440, {0x5b51038, 0xc0011cec70}) + :1 +0xaf fp=0xc0010f3370 sp=0xc0010f3320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000bfd440}, {0x5b4a6f0?, 0xc00043b340}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0010f33b0 sp=0xc0010f3370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc0011ecd80}, 0xc000a738c0?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0010f3400 sp=0xc0010f33b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc0011ecd80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0010f3490 sp=0xc0010f3400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc000e84f00}, 0xc000a738c0, 0xc0011ca980) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0010f37d0 sp=0xc0010f3490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc000e84f00}, 0x3f252a0?, 0xc0011ceba0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0010f3848 sp=0xc0010f37d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc001197320, 0xc000c4a570, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0010f3db0 sp=0xc0010f3848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc001197320, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0010f3f28 sp=0xc0010f3db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0010f3fe0 sp=0xc0010f3f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010f3fe8 sp=0xc0010f3fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 559 [select]: +runtime.gopark(0xc000f77e38?, 0x2?, 0xb4?, 0x1b?, 0xc000f77c5c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029b9a58 sp=0xc0029b9a38 pc=0x4777ae +runtime.selectgo(0xc0029b9e38, 0xc000f77c58, 0x0?, 0x0, 0x4113100?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029b9ba8 sp=0xc0029b9a58 pc=0x48916b +github.com/rancher/opni/pkg/gateway.(*GatewayGRPCServer).ListenAndServe(0xc000c95da0, {0x5b426e0, 0xc0020885a0}) + /home/ec2-user/opni/pkg/gateway/grpc.go:107 +0x106f fp=0xc0029b9e68 sp=0xc0029b9ba8 pc=0x22b4b0f +github.com/rancher/opni/pkg/gateway.(*Gateway).ListenAndServe.func2() + /home/ec2-user/opni/pkg/gateway/gateway.go:338 +0x86 fp=0xc0029b9fa0 sp=0xc0029b9e68 pc=0x22b2686 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0029b9fe0 sp=0xc0029b9fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029b9fe8 sp=0xc0029b9fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 581 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 428 [chan receive]: +runtime.gopark(0xc000b3d020?, 0x27?, 0x0?, 0x0?, 0x460fed5?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00148bef8 sp=0xc00148bed8 pc=0x4777ae +runtime.chanrecv(0xc0010886c0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00148bf70 sp=0xc00148bef8 pc=0x441325 +runtime.chanrecv1(0xc000762460?, 0x47e975?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00148bf98 sp=0xc00148bf70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc00148bfe0 sp=0xc00148bf98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00148bfe8 sp=0xc00148bfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 424 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 343 [select]: +runtime.gopark(0xc002499e70?, 0x2?, 0x0?, 0x0?, 0xc002499df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002499c58 sp=0xc002499c38 pc=0x4777ae +runtime.selectgo(0xc002499e70, 0xc002499df0, 0xc002499e00?, 0x0, 0xc0010b8700?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002499da8 sp=0xc002499c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0007c17c0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002499ea0 sp=0xc002499da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0006109a0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002499f58 sp=0xc002499ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc002499fe0 sp=0xc002499f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002499fe8 sp=0xc002499fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 323 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 351 [select]: +runtime.gopark(0xc0010f6890?, 0x2?, 0x98?, 0x67?, 0xc0010f683c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010f66a8 sp=0xc0010f6688 pc=0x4777ae +runtime.selectgo(0xc0010f6890, 0xc0010f6838, 0x7f429a427900?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010f67f8 sp=0xc0010f66a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc001434190, {0xc001472d30, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0010f68c0 sp=0xc0010f67f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001434190, {0xc001472d30, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0010f6958 sp=0xc0010f68c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0012672f0, {0xc001472d30, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0010f69a0 sp=0xc0010f6958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0012672f0}, {0xc001472d30, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0010f69e8 sp=0xc0010f69a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00123d200, {0xc001472d30, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0010f6a58 sp=0xc0010f69e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001472d20, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0010f6b28 sp=0xc0010f6a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc00141d440?, 0xc00123d200, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0010f6c18 sp=0xc0010f6b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001472d20, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001434640}, 0xc000c4a360?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0010f6cd0 sp=0xc0010f6c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc0012341e0, {0x43dcf60?, 0xc001434640}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0010f6f08 sp=0xc0010f6cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001472e70, {0x43dcf60, 0xc001434640}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0010f6ff0 sp=0xc0010f6f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc00127d2a0, {0x43dcf60, 0xc001434640}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0010f7028 sp=0xc0010f6ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00158a120) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc0010f7068 sp=0xc0010f7028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00158a120?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0010f70a8 sp=0xc0010f7068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00158a130) + :1 +0x46 fp=0xc0010f70e0 sp=0xc0010f70a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c4a4e0, {0x5b51368, 0xc00158a130}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc0010f7320 sp=0xc0010f70e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000bfd440, {0x5b51038, 0xc00158a120}) + :1 +0xaf fp=0xc0010f7370 sp=0xc0010f7320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000bfd440}, {0x5b4a6f0?, 0xc00127d2a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0010f73b0 sp=0xc0010f7370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc001472e70}, 0xc001422db0?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0010f7400 sp=0xc0010f73b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc001472e70}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0010f7490 sp=0xc0010f7400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc0012341e0}, 0xc001422db0, 0xc00141ca40) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0010f77d0 sp=0xc0010f7490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc0012341e0}, 0x3f252a0?, 0xc00158a050?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0010f7848 sp=0xc0010f77d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc00123d200, 0xc000c4a570, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0010f7db0 sp=0xc0010f7848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc00123d200, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0010f7f28 sp=0xc0010f7db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0010f7fe0 sp=0xc0010f7f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010f7fe8 sp=0xc0010f7fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 284 [IO wait]: +runtime.gopark(0xc0010ecfe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010ecf98 sp=0xc0010ecf78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0010ecfd0 sp=0xc0010ecf98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42416777e8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0010ecff0 sp=0xc0010ecfd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0010e02a0, 0xc00139e600?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0010ed038 sp=0xc0010ecff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0010e0280, {0xc00139e600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0010ed128 sp=0xc0010ed038 pc=0x50e5a5 +net.(*netFD).Read(0xc0010e0280, {0xc00139e600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0010ed188 sp=0xc0010ed128 pc=0x89342b +net.(*conn).Read(0xc00007de20, {0xc00139e600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0010ed210 sp=0xc0010ed188 pc=0x8b176d +net.(*UnixConn).Read(0xc00007de20, {0xc00139e600, 0x600, 0x600}) + :1 +0x54 fp=0xc0010ed258 sp=0xc0010ed210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc000f1ffe0, {0xc00139e600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0010ed2b8 sp=0xc0010ed258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0010b8d28, {0x5b0f6c0, 0xc000f1ffe0}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0010ed320 sp=0xc0010ed2b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc0010b8a80, {0x5b0f840?, 0xc00007de20}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc0010ed390 sp=0xc0010ed320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc0010b8a80, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0010ed9c0 sp=0xc0010ed390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc0010b8a80, {0xc0013a6000, 0x8000, 0x4?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc0010edaa8 sp=0xc0010ed9c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0009d3c40, {0xc0013a6000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc0010edaf0 sp=0xc0010edaa8 pc=0x10969ed +bufio.(*Reader).Read(0xc0013993e0, {0xc000c49620, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0010edbb0 sp=0xc0010edaf0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0013993e0}, {0xc000c49620, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0010edbf8 sp=0xc0010edbb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000c49620, 0x9, 0x9}, {0x5b0f0a0, 0xc0013993e0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0010edc98 sp=0xc0010edbf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000c495e0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0010eddd0 sp=0xc0010edc98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001128240, 0xc000c46e50?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0010edfb0 sp=0xc0010eddd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0010edfe0 sp=0xc0010edfb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010edfe8 sp=0xc0010edfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 322 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 558 [select]: +runtime.gopark(0xc0017dd140?, 0x3?, 0xa0?, 0xea?, 0xc00209dd9a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029c57d0 sp=0xc0029c57b0 pc=0x4777ae +runtime.selectgo(0xc0017dd140, 0xc00209dd94, 0x0?, 0x0, 0xc001bf1ad0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029c5920 sp=0xc0029c57d0 pc=0x48916b +reflect.rselect({0xc0029c5b08, 0x3, 0x7f4241663488?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:589 +0x345 fp=0xc0029c59d0 sp=0xc0029c5920 pc=0x4a8f05 +reflect.Select({0xc00163bea0, 0x3, 0x2?}) + /home/ec2-user/sdk/go1.21.0/src/reflect/value.go:3104 +0xb8a fp=0xc0029c5b98 sp=0xc0029c59d0 pc=0x57c44a +github.com/rancher/opni/pkg/util.WaitAll({0x5b426e0, 0xc001fee370}, 0xc001bf1a20, {0xc0029c5e08, 0x2, 0x2}) + /home/ec2-user/opni/pkg/util/servers.go:51 +0x3ea fp=0xc0029c5d18 sp=0xc0029c5b98 pc=0x176ae8a +github.com/rancher/opni/pkg/gateway.(*GatewayHTTPServer).ListenAndServe(0xc0008ab5c0, {0x5b426e0, 0xc0020885a0}) + /home/ec2-user/opni/pkg/gateway/http.go:185 +0x5a7 fp=0xc0029c5e68 sp=0xc0029c5d18 pc=0x22b8427 +github.com/rancher/opni/pkg/gateway.(*Gateway).ListenAndServe.func1() + /home/ec2-user/opni/pkg/gateway/gateway.go:327 +0x86 fp=0xc0029c5fa0 sp=0xc0029c5e68 pc=0x22b28a6 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0029c5fe0 sp=0xc0029c5fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029c5fe8 sp=0xc0029c5fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 581 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 283 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011178d8 sp=0xc0011178b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001117910 sp=0xc0011178d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42416775f8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001117930 sp=0xc001117910 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000dbb3a0, 0xc001170001?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001117978 sp=0xc001117930 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc000dbb380) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001117a68 sp=0xc001117978 pc=0x513f65 +net.(*netFD).accept(0xc000dbb380) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001117b60 sp=0xc001117a68 pc=0x8967be +net.(*TCPListener).accept(0xc0009d26e0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc001117ba8 sp=0xc001117b60 pc=0x8c383e +net.(*TCPListener).Accept(0xc0009d26e0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc001117c18 sp=0xc001117ba8 pc=0x8c1ba5 +crypto/tls.(*listener).Accept(0xc000dc3440) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/tls.go:66 +0x43 fp=0xc001117c70 sp=0xc001117c18 pc=0x9b2d43 +net/http.(*onceCloseListener).Accept(0xc0011511d0) + :1 +0x44 fp=0xc001117ca8 sp=0xc001117c70 pc=0xab5544 +net/http.(*Server).Serve(0xc000c07590, {0x5b2a5b0, 0xc000dc3440}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc001117ea0 sp=0xc001117ca8 pc=0xa7859f +net/http.Serve(...) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:2595 +github.com/gin-gonic/gin.(*Engine).RunListener(0xc000c7c4e0, {0x5b2a5b0, 0xc000dc3440}) + /home/ec2-user/go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/gin.go:565 +0x370 fp=0xc001117f90 sp=0xc001117ea0 pc=0x171e870 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure.func1() + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:72 +0x5c fp=0xc001117fe0 sp=0xc001117f90 pc=0x262a57c +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001117fe8 sp=0xc001117fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure in goroutine 279 + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:71 +0x26d + +goroutine 311 [select]: +runtime.gopark(0xc0012aafb0?, 0x2?, 0xb4?, 0x1b?, 0xc0012aaf74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012aade8 sp=0xc0012aadc8 pc=0x4777ae +runtime.selectgo(0xc0012aafb0, 0xc0012aaf70, 0x4aead9?, 0x0, 0x4ad421?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0012aaf38 sp=0xc0012aade8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0012aafe0 sp=0xc0012aaf38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012aafe8 sp=0xc0012aafe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 312 [select]: +runtime.gopark(0xc001241fb0?, 0x2?, 0xb0?, 0x1d?, 0xc001241ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001241d58 sp=0xc001241d38 pc=0x4777ae +runtime.selectgo(0xc001241fb0, 0xc001241ee0, 0x1?, 0x0, 0x4b0571?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001241ea8 sp=0xc001241d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001241fe0 sp=0xc001241ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001241fe8 sp=0xc001241fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 313 [chan receive]: +runtime.gopark(0x77000000004b0571?, 0x12a7458?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00119cd60 sp=0xc00119cd40 pc=0x4777ae +runtime.chanrecv(0xc00119aa80, 0xc00119ce60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00119cdd8 sp=0xc00119cd60 pc=0x441325 +runtime.chanrecv1(0x28?, 0x14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00119ce00 sp=0xc00119cdd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc00119cfe0 sp=0xc00119ce00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00119cfe8 sp=0xc00119cfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 364 [select]: +runtime.gopark(0xc0011dbfb0?, 0x2?, 0xb4?, 0x1b?, 0xc0011dbf74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011dbde8 sp=0xc0011dbdc8 pc=0x4777ae +runtime.selectgo(0xc0011dbfb0, 0xc0011dbf70, 0xc00128b440?, 0x0, 0xc001286d80?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011dbf38 sp=0xc0011dbde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0011dbfe0 sp=0xc0011dbf38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011dbfe8 sp=0xc0011dbfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 329 [select]: +runtime.gopark(0xc001294fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001294f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001294de8 sp=0xc001294dc8 pc=0x4777ae +runtime.selectgo(0xc001294fb0, 0xc001294f70, 0x4aead9?, 0x0, 0x4ad421?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001294f38 sp=0xc001294de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001294fe0 sp=0xc001294f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001294fe8 sp=0xc001294fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 330 [select]: +runtime.gopark(0xc001295fb0?, 0x2?, 0xb0?, 0x5d?, 0xc001295ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001295d58 sp=0xc001295d38 pc=0x4777ae +runtime.selectgo(0xc001295fb0, 0xc001295ee0, 0x448855?, 0x0, 0x400a060?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001295ea8 sp=0xc001295d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001295fe0 sp=0xc001295ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001295fe8 sp=0xc001295fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 331 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001296d60 sp=0xc001296d40 pc=0x4777ae +runtime.chanrecv(0xc001286240, 0xc001296e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001296dd8 sp=0xc001296d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001296e00 sp=0xc001296dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001296fe0 sp=0xc001296e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001296fe8 sp=0xc001296fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 332 [select]: +runtime.gopark(0xc0011fa890?, 0x2?, 0x98?, 0xa7?, 0xc0011fa83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011fa6a8 sp=0xc0011fa688 pc=0x4777ae +runtime.selectgo(0xc0011fa890, 0xc0011fa838, 0x7f429a41c800?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011fa7f8 sp=0xc0011fa6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00128c0a0, {0xc0010e4cd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0011fa8c0 sp=0xc0011fa7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00128c0a0, {0xc0010e4cd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0011fa958 sp=0xc0011fa8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0010e4c30, {0xc0010e4cd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0011fa9a0 sp=0xc0011fa958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0010e4c30}, {0xc0010e4cd0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0011fa9e8 sp=0xc0011fa9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0010dcb40, {0xc0010e4cd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0011faa58 sp=0xc0011fa9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0010e4cc0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0011fab28 sp=0xc0011faa58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001282e00?, 0xc0010dcb40, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0011fac18 sp=0xc0011fab28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0010e4cc0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc00128c3c0}, 0xc000c70240?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0011facd0 sp=0xc0011fac18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc0010a6690, {0x43dcf60?, 0xc00128c3c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0011faf08 sp=0xc0011facd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0010e4e10, {0x43dcf60, 0xc00128c3c0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0011faff0 sp=0xc0011faf08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc000b30640, {0x43dcf60, 0xc00128c3c0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0011fb028 sp=0xc0011faff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc0010b5f90) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc0011fb068 sp=0xc0011fb028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc0010b5f90?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0011fb0a8 sp=0xc0011fb068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc0010b5fa0) + :1 +0x46 fp=0xc0011fb0e0 sp=0xc0011fb0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c703c0, {0x5b51368, 0xc0010b5fa0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc0011fb320 sp=0xc0011fb0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000c6e5e0, {0x5b51038, 0xc0010b5f90}) + :1 +0xaf fp=0xc0011fb370 sp=0xc0011fb320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000c6e5e0}, {0x5b4a6f0?, 0xc000b30640}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0011fb3b0 sp=0xc0011fb370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc0010e4e10}, 0xc0010d8588?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0011fb400 sp=0xc0011fb3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc0010e4e10}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0011fb490 sp=0xc0011fb400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc0010a6690}, 0xc0010d8588, 0xc001282200) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0011fb7d0 sp=0xc0011fb490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc0010a6690}, 0x3f252a0?, 0xc0010b5ec0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0011fb848 sp=0xc0011fb7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc0010dcb40, 0xc000c70450, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0011fbdb0 sp=0xc0011fb848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc0010dcb40, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0011fbf28 sp=0xc0011fbdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0011fbfe0 sp=0xc0011fbf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011fbfe8 sp=0xc0011fbfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 365 [select]: +runtime.gopark(0xc0012b0fb0?, 0x2?, 0xb0?, 0xd?, 0xc0012b0ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012b0d58 sp=0xc0012b0d38 pc=0x4777ae +runtime.selectgo(0xc0012b0fb0, 0xc0012b0ee0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0012b0ea8 sp=0xc0012b0d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0012b0fe0 sp=0xc0012b0ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012b0fe8 sp=0xc0012b0fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1156 [select]: +runtime.gopark(0xc002d29f90?, 0x2?, 0x30?, 0x9d?, 0xc002d29e94?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d29ce8 sp=0xc002d29cc8 pc=0x4777ae +runtime.selectgo(0xc002d29f90, 0xc002d29e90, 0xc00263dbeb?, 0x0, 0xc001487ea8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d29e38 sp=0xc002d29ce8 pc=0x48916b +github.com/rancher/opni/pkg/storage/jetstream.(*JetStreamStore).WatchClusters.func1() + /home/ec2-user/opni/pkg/storage/jetstream/cluster_store.go:247 +0x18c fp=0xc002d29fe0 sp=0xc002d29e38 pc=0x249e4ac +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d29fe8 sp=0xc002d29fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage/jetstream.(*JetStreamStore).WatchClusters in goroutine 1133 + /home/ec2-user/opni/pkg/storage/jetstream/cluster_store.go:244 +0xbcd + +goroutine 335 [select]: +runtime.gopark(0xc001293fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001293f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001293de8 sp=0xc001293dc8 pc=0x4777ae +runtime.selectgo(0xc001293fb0, 0xc001293f70, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001293f38 sp=0xc001293de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001293fe0 sp=0xc001293f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001293fe8 sp=0xc001293fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 336 [select]: +runtime.gopark(0xc001292fb0?, 0x2?, 0xb0?, 0x2d?, 0xc001292ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001292d58 sp=0xc001292d38 pc=0x4777ae +runtime.selectgo(0xc001292fb0, 0xc001292ee0, 0x448855?, 0x0, 0x400a060?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001292ea8 sp=0xc001292d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001292fe0 sp=0xc001292ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001292fe8 sp=0xc001292fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 337 [chan receive]: +runtime.gopark(0xc0010b5cb0?, 0xc00070f822?, 0xe?, 0x0?, 0x5b058a8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001297d60 sp=0xc001297d40 pc=0x4777ae +runtime.chanrecv(0xc0012867e0, 0xc001297e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001297dd8 sp=0xc001297d60 pc=0x441325 +runtime.chanrecv1(0xc0010b5cb0?, 0xc00070f822?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001297e00 sp=0xc001297dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001297fe0 sp=0xc001297e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001297fe8 sp=0xc001297fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 354 [select]: +runtime.gopark(0xc0012a8890?, 0x2?, 0x98?, 0x87?, 0xc0012a883c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012a86a8 sp=0xc0012a8688 pc=0x4777ae +runtime.selectgo(0xc0012a8890, 0xc0012a8838, 0x7f429a42b200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0012a87f8 sp=0xc0012a86a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00128cb40, {0xc0010e5d50, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0012a88c0 sp=0xc0012a87f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00128cb40, {0xc0010e5d50, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0012a8958 sp=0xc0012a88c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0010e56e0, {0xc0010e5d50, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0012a89a0 sp=0xc0012a8958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0010e56e0}, {0xc0010e5d50, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0012a89e8 sp=0xc0012a89a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0010dd440, {0xc0010e5d50, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0012a8a58 sp=0xc0012a89e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0010e5d40, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0012a8b28 sp=0xc0012a8a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0012b4500?, 0xc0010dd440, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0012a8c18 sp=0xc0012a8b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0010e5d40, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc00128d400}, 0xc000c4a360?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0012a8cd0 sp=0xc0012a8c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc0010a6780, {0x43dcf60?, 0xc00128d400}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0012a8f08 sp=0xc0012a8cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0010e5e90, {0x43dcf60, 0xc00128d400}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0012a8ff0 sp=0xc0012a8f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc000b30aa0, {0x43dcf60, 0xc00128d400}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0012a9028 sp=0xc0012a8ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00128e680) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc0012a9068 sp=0xc0012a9028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00128e680?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0012a90a8 sp=0xc0012a9068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00128e690) + :1 +0x46 fp=0xc0012a90e0 sp=0xc0012a90a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c4a4e0, {0x5b51368, 0xc00128e690}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc0012a9320 sp=0xc0012a90e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000bfd440, {0x5b51038, 0xc00128e680}) + :1 +0xaf fp=0xc0012a9370 sp=0xc0012a9320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000bfd440}, {0x5b4a6f0?, 0xc000b30aa0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0012a93b0 sp=0xc0012a9370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc0010e5e90}, 0xc0010d8c78?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0012a9400 sp=0xc0012a93b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc0010e5e90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0012a9490 sp=0xc0012a9400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc0010a6780}, 0xc0010d8c78, 0xc001283b00) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0012a97d0 sp=0xc0012a9490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc0010a6780}, 0x3f252a0?, 0xc00128e5b0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0012a9848 sp=0xc0012a97d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc0010dd440, 0xc000c4a570, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0012a9db0 sp=0xc0012a9848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc0010dd440, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0012a9f28 sp=0xc0012a9db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0012a9fe0 sp=0xc0012a9f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012a9fe8 sp=0xc0012a9fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 358 [select]: +runtime.gopark(0xc001291fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001291f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001291de8 sp=0xc001291dc8 pc=0x4777ae +runtime.selectgo(0xc001291fb0, 0xc001291f70, 0xc00128ad20?, 0x0, 0xc0012869c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001291f38 sp=0xc001291de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001291fe0 sp=0xc001291f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001291fe8 sp=0xc001291fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 359 [select]: +runtime.gopark(0xc0012affb0?, 0x2?, 0xb0?, 0xfd?, 0xc0012afee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012afd58 sp=0xc0012afd38 pc=0x4777ae +runtime.selectgo(0xc0012affb0, 0xc0012afee0, 0x448855?, 0x0, 0x4166b20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0012afea8 sp=0xc0012afd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0012affe0 sp=0xc0012afea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012affe8 sp=0xc0012affe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 360 [chan receive]: +runtime.gopark(0x7700000000000000?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012aed60 sp=0xc0012aed40 pc=0x4777ae +runtime.chanrecv(0xc001286c60, 0xc0012aee60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0012aedd8 sp=0xc0012aed60 pc=0x441325 +runtime.chanrecv1(0x28?, 0x14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0012aee00 sp=0xc0012aedd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc0012aefe0 sp=0xc0012aee00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012aefe8 sp=0xc0012aefe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 361 [select]: +runtime.gopark(0xc0012a4890?, 0x2?, 0x98?, 0x47?, 0xc0012a483c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012a46a8 sp=0xc0012a4688 pc=0x4777ae +runtime.selectgo(0xc0012a4890, 0xc0012a4838, 0x7f429a424300?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0012a47f8 sp=0xc0012a46a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00128d900, {0xc0012b67c0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0012a48c0 sp=0xc0012a47f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00128d900, {0xc0012b67c0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0012a4958 sp=0xc0012a48c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0012b6720, {0xc0012b67c0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0012a49a0 sp=0xc0012a4958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0012b6720}, {0xc0012b67c0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0012a49e8 sp=0xc0012a49a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0010dd9e0, {0xc0012b67c0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0012a4a58 sp=0xc0012a49e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0012b67b0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0012a4b28 sp=0xc0012a4a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0012b5a00?, 0xc0010dd9e0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0012a4c18 sp=0xc0012a4b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0012b67b0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc00128dc20}, 0xc000c70240?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0012a4cd0 sp=0xc0012a4c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc0010a6870, {0x43dcf60?, 0xc00128dc20}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0012a4f08 sp=0xc0012a4cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0012b6900, {0x43dcf60, 0xc00128dc20}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0012a4ff0 sp=0xc0012a4f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc000b30ce0, {0x43dcf60, 0xc00128dc20}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0012a5028 sp=0xc0012a4ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00128eaa0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc0012a5068 sp=0xc0012a5028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00128eaa0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0012a50a8 sp=0xc0012a5068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00128eab0) + :1 +0x46 fp=0xc0012a50e0 sp=0xc0012a50a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c703c0, {0x5b51368, 0xc00128eab0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc0012a5320 sp=0xc0012a50e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000c6e5e0, {0x5b51038, 0xc00128eaa0}) + :1 +0xaf fp=0xc0012a5370 sp=0xc0012a5320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000c6e5e0}, {0x5b4a6f0?, 0xc000b30ce0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0012a53b0 sp=0xc0012a5370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc0012b6900}, 0xc0010d90e0?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0012a5400 sp=0xc0012a53b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc0012b6900}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0012a5490 sp=0xc0012a5400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc0010a6870}, 0xc0010d90e0, 0xc0012b4e00) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0012a57d0 sp=0xc0012a5490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc0010a6870}, 0x3f252a0?, 0xc00128e9d0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0012a5848 sp=0xc0012a57d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc0010dd9e0, 0xc000c70450, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0012a5db0 sp=0xc0012a5848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc0010dd9e0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0012a5f28 sp=0xc0012a5db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0012a5fe0 sp=0xc0012a5f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012a5fe8 sp=0xc0012a5fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 288 [IO wait]: +runtime.gopark(0xc001140f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001140f40 sp=0xc001140f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001140f78 sp=0xc001140f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42416776f0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001140f98 sp=0xc001140f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d179a0, 0xc000d1d680?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001140fe0 sp=0xc001140f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000d17980, {0xc000d1d680, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0011410d0 sp=0xc001140fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc000d17980, {0xc000d1d680, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001141130 sp=0xc0011410d0 pc=0x89342b +net.(*conn).Read(0xc000991360, {0xc000d1d680, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0011411b8 sp=0xc001141130 pc=0x8b176d +net.(*UnixConn).Read(0xc000991360, {0xc000d1d680, 0x240, 0x240}) + :1 +0x54 fp=0xc001141200 sp=0xc0011411b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc001422108, {0xc000d1d680, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001141260 sp=0xc001141200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc00125e2a8, {0x5b0f6c0, 0xc001422108}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0011412c8 sp=0xc001141260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc00125e000, {0x5b0f840?, 0xc000991360}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001141338 sp=0xc0011412c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc00125e000, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001141968 sp=0xc001141338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc00125e000, {0xc0013be000, 0x8000, 0x0?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001141a50 sp=0xc001141968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0009d3cc0, {0xc0013be000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001141a98 sp=0xc001141a50 pc=0x10969ed +bufio.(*Reader).Read(0xc0013996e0, {0xc000c49700, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001141b58 sp=0xc001141a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0013996e0}, {0xc000c49700, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001141ba0 sp=0xc001141b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000c49700, 0x9, 0x9}, {0x5b0f0a0, 0xc0013996e0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001141c40 sp=0xc001141ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000c496c0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001141d78 sp=0xc001141c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001174340, 0x4b05a9?, 0xc000b81ef8?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001141ec8 sp=0xc001141d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc0004ef680, {0x5b54e00?, 0xc001174340}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001141f80 sp=0xc001141ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001141fe0 sp=0xc001141f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001141fe8 sp=0xc001141fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 304 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 402 [select]: +runtime.gopark(0xc0011d9fb0?, 0x2?, 0xb4?, 0x1b?, 0xc0011d9f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011d9de8 sp=0xc0011d9dc8 pc=0x4777ae +runtime.selectgo(0xc0011d9fb0, 0xc0011d9f70, 0x4b05a9?, 0x0, 0x8d0f55?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011d9f38 sp=0xc0011d9de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0011d9fe0 sp=0xc0011d9f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011d9fe8 sp=0xc0011d9fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 353 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 366 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012b1d60 sp=0xc0012b1d40 pc=0x4777ae +runtime.chanrecv(0xc001287020, 0xc0012b1e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0012b1dd8 sp=0xc0012b1d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0012b1e00 sp=0xc0012b1dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc0012b1fe0 sp=0xc0012b1e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012b1fe8 sp=0xc0012b1fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 409 [select]: +runtime.gopark(0xc00142be48?, 0x2?, 0x0?, 0x0?, 0xc00142bdcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029c7c30 sp=0xc0029c7c10 pc=0x4777ae +runtime.selectgo(0xc0029c7e48, 0xc00142bdc8, 0xc00142bdd8?, 0x0, 0xc000dc1500?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029c7d80 sp=0xc0029c7c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001434be0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0029c7e78 sp=0xc0029c7d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000611110) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0029c7f30 sp=0xc0029c7e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc0029c7fe0 sp=0xc0029c7f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029c7fe8 sp=0xc0029c7fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 379 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 370 [select]: +runtime.gopark(0xc00115ef70?, 0x4?, 0x0?, 0x0?, 0xc00115edd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00115ec00 sp=0xc00115ebe0 pc=0x4777ae +runtime.selectgo(0xc00115ef70, 0xc00115edd0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00115ed50 sp=0xc00115ec00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001174680) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc00115efc0 sp=0xc00115ed50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc00115efe0 sp=0xc00115efc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00115efe8 sp=0xc00115efe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 303 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 371 [IO wait]: +runtime.gopark(0xc001268f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001268f40 sp=0xc001268f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001268f78 sp=0xc001268f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42416778e0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001268f98 sp=0xc001268f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d17820, 0xc000d1d200?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001268fe0 sp=0xc001268f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000d17800, {0xc000d1d200, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0012690d0 sp=0xc001268fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc000d17800, {0xc000d1d200, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001269130 sp=0xc0012690d0 pc=0x89342b +net.(*conn).Read(0xc000991358, {0xc000d1d200, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0012691b8 sp=0xc001269130 pc=0x8b176d +net.(*UnixConn).Read(0xc000991358, {0xc000d1d200, 0x240, 0x240}) + :1 +0x54 fp=0xc001269200 sp=0xc0012691b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0014224c8, {0xc000d1d200, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001269260 sp=0xc001269200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000f89ea8, {0x5b0f6c0, 0xc0014224c8}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0012692c8 sp=0xc001269260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000f89c00, {0x5b0f840?, 0xc000991358}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001269338 sp=0xc0012692c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000f89c00, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001269968 sp=0xc001269338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000f89c00, {0xc0013d6000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001269a50 sp=0xc001269968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0009d3d40, {0xc0013d6000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001269a98 sp=0xc001269a50 pc=0x10969ed +bufio.(*Reader).Read(0xc001399980, {0xc000c497e0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001269b58 sp=0xc001269a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001399980}, {0xc000c497e0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001269ba0 sp=0xc001269b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000c497e0, 0x9, 0x9}, {0x5b0f0a0, 0xc001399980}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001269c40 sp=0xc001269ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000c497a0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001269d78 sp=0xc001269c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001174680, 0x8d0f0a?, 0x87c25f?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001269ec8 sp=0xc001269d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc0004ef860, {0x5b54e00?, 0xc001174680}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001269f80 sp=0xc001269ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001269fe0 sp=0xc001269f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001269fe8 sp=0xc001269fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 303 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1352 [select]: +runtime.gopark(0xc0022d9f70?, 0x2?, 0xb4?, 0x1b?, 0xc0022d9ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0022d9d20 sp=0xc0022d9d00 pc=0x4777ae +runtime.selectgo(0xc0022d9f70, 0xc0022d9eb8, 0xc001487f80?, 0x0, 0x100000001?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0022d9e70 sp=0xc0022d9d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002180180, {0x5b426e0, 0xc001abb450}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0022d9fa0 sp=0xc0022d9e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0022d9fe0 sp=0xc0022d9fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0022d9fe8 sp=0xc0022d9fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1320 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 580 [select]: +runtime.gopark(0xc0020b8b70?, 0x3?, 0x49?, 0x0?, 0xc001fe007a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001290810 sp=0xc0012907f0 pc=0x4777ae +runtime.selectgo(0xc0020b8b70, 0xc001fe0074, 0x0?, 0x0, 0xc0020843f0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001290960 sp=0xc001290810 pc=0x48916b +reflect.rselect({0xc001290b48, 0x3, 0x447c9e?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:589 +0x345 fp=0xc001290a10 sp=0xc001290960 pc=0x4a8f05 +reflect.Select({0xc001eff7a0, 0x3, 0x2?}) + /home/ec2-user/sdk/go1.21.0/src/reflect/value.go:3104 +0xb8a fp=0xc001290bd8 sp=0xc001290a10 pc=0x57c44a +github.com/rancher/opni/pkg/util.WaitAll({0x5b426e0, 0xc001152370}, 0xc002084180, {0xc001290db8, 0x2, 0x2}) + /home/ec2-user/opni/pkg/util/servers.go:51 +0x3ea fp=0xc001290d58 sp=0xc001290bd8 pc=0x176ae8a +github.com/rancher/opni/pkg/management.(*Server).ListenAndServe(0xc000d8a000, {0x5b426e0, 0xc000b69f40}) + /home/ec2-user/opni/pkg/management/server.go:196 +0x208 fp=0xc001290dd8 sp=0xc001290d58 pc=0x231df08 +github.com/rancher/opni/pkg/test.(*Environment).startGateway.func2(0x4b0571) + /home/ec2-user/opni/pkg/test/environment.go:1699 +0x18c fp=0xc001290f88 sp=0xc001290dd8 pc=0x24c574c +github.com/rancher/opni/pkg/plugins/hooks.onLoadingCompletedHook.Invoke(0xc0007d30e0, 0xc001290fd0?) + /home/ec2-user/opni/pkg/plugins/hooks/complete.go:10 +0x34 fp=0xc001290fa8 sp=0xc001290f88 pc=0x2182b94 +github.com/rancher/opni/pkg/plugins.(*PluginLoader).Complete.func1() + /home/ec2-user/opni/pkg/plugins/loader.go:329 +0x51 fp=0xc001290fe0 sp=0xc001290fa8 pc=0x2189711 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001290fe8 sp=0xc001290fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins.(*PluginLoader).Complete in goroutine 65 + /home/ec2-user/opni/pkg/plugins/loader.go:329 +0x138 + +goroutine 373 [select]: +runtime.gopark(0xc00115dfb0?, 0x2?, 0xb4?, 0x1b?, 0xc00115df74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00115dde8 sp=0xc00115ddc8 pc=0x4777ae +runtime.selectgo(0xc00115dfb0, 0xc00115df70, 0x4ad421?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00115df38 sp=0xc00115dde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc00115dfe0 sp=0xc00115df38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00115dfe8 sp=0xc00115dfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 374 [select]: +runtime.gopark(0xc001115fb0?, 0x2?, 0xb0?, 0x5d?, 0xc001115ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001115d58 sp=0xc001115d38 pc=0x4777ae +runtime.selectgo(0xc001115fb0, 0xc001115ee0, 0x448855?, 0x0, 0x4166b20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001115ea8 sp=0xc001115d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001115fe0 sp=0xc001115ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001115fe8 sp=0xc001115fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 375 [chan receive]: +runtime.gopark(0x7700000000000000?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00115fd60 sp=0xc00115fd40 pc=0x4777ae +runtime.chanrecv(0xc001142f60, 0xc00115fe60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00115fdd8 sp=0xc00115fd60 pc=0x441325 +runtime.chanrecv1(0x28?, 0x14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00115fe00 sp=0xc00115fdd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc00115ffe0 sp=0xc00115fe00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00115ffe8 sp=0xc00115ffe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 387 [select]: +runtime.gopark(0xc00126bf70?, 0x2?, 0xb4?, 0x1b?, 0xc00126bebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00126bd20 sp=0xc00126bd00 pc=0x4777ae +runtime.selectgo(0xc00126bf70, 0xc00126beb8, 0x4b3745?, 0x0, 0xc000bdcb08?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00126be70 sp=0xc00126bd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b31240, {0x5b426e0, 0xc0012c2820}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc00126bfa0 sp=0xc00126be70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc00126bfe0 sp=0xc00126bfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00126bfe8 sp=0xc00126bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 307 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 388 [select]: +runtime.gopark(0xc00113df70?, 0x2?, 0xb4?, 0x1b?, 0xc00113debc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00113dd20 sp=0xc00113dd00 pc=0x4777ae +runtime.selectgo(0xc00113df70, 0xc00113deb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00113de70 sp=0xc00113dd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b312a0, {0x5b426e0, 0xc0012c2870}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc00113dfa0 sp=0xc00113de70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc00113dfe0 sp=0xc00113dfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00113dfe8 sp=0xc00113dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 307 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 385 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x26?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018d1bd0 sp=0xc0018d1bb0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0018d1c08 sp=0xc0018d1bd0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162ab58, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0018d1c28 sp=0xc0018d1c08 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000a3df20, 0xc0019cc000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0018d1c70 sp=0xc0018d1c28 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000a3df00, {0xc0019cc000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0018d1d60 sp=0xc0018d1c70 pc=0x50e5a5 +net.(*netFD).Read(0xc000a3df00, {0xc0019cc000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0018d1dc0 sp=0xc0018d1d60 pc=0x89342b +net.(*conn).Read(0xc000b78758, {0xc0019cc000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0018d1e48 sp=0xc0018d1dc0 pc=0x8b176d +net.(*TCPConn).Read(0xc000b78758, {0xc0019cc000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc0018d1e90 sp=0xc0018d1e48 pc=0x8d72f4 +github.com/nats-io/nats%2ego.(*natsReader).Read(0xc000fe7c00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1890 +0x1a9 fp=0xc0018d1f10 sp=0xc0018d1e90 pc=0x1f6d169 +github.com/nats-io/nats%2ego.(*Conn).readLoop(0xc000a46000) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2917 +0x1f8 fp=0xc0018d1fc0 sp=0xc0018d1f10 pc=0x1f75fb8 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x34 fp=0xc0018d1fe0 sp=0xc0018d1fc0 pc=0x1f71514 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018d1fe8 sp=0xc0018d1fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 177 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x44e + +goroutine 429 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015f5978 sp=0xc0015f5958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0015f59b0 sp=0xc0015f5978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677310, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0015f59d0 sp=0xc0015f59b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00144f020, 0xc0015f5a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0015f5a18 sp=0xc0015f59d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc00144f000) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0015f5b08 sp=0xc0015f5a18 pc=0x513f65 +net.(*netFD).accept(0xc00144f000) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0015f5c00 sp=0xc0015f5b08 pc=0x8967be +net.(*UnixListener).accept(0xc0015deab0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc0015f5c40 sp=0xc0015f5c00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc0015deab0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc0015f5cb0 sp=0xc0015f5c40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc00127db80) + :1 +0x44 fp=0xc0015f5ce8 sp=0xc0015f5cb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc00141a3c0, {0x5b2c6a0, 0xc00127db80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc0015f5f18 sp=0xc0015f5ce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc0015f5f68 sp=0xc0015f5f18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc00127de00?, 0xc00158afe0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc0015f5fa8 sp=0xc0015f5f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc0015f5fe0 sp=0xc0015f5fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015f5fe8 sp=0xc0015f5fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 424 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 380 [IO wait]: +runtime.gopark(0xc001498fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001498f98 sp=0xc001498f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001498fd0 sp=0xc001498f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677500, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001498ff0 sp=0xc001498fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000dbbca0, 0xc00139f800?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001499038 sp=0xc001498ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000dbbc80, {0xc00139f800, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001499128 sp=0xc001499038 pc=0x50e5a5 +net.(*netFD).Read(0xc000dbbc80, {0xc00139f800, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001499188 sp=0xc001499128 pc=0x89342b +net.(*conn).Read(0xc000640318, {0xc00139f800, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001499210 sp=0xc001499188 pc=0x8b176d +net.(*UnixConn).Read(0xc000640318, {0xc00139f800, 0x600, 0x600}) + :1 +0x54 fp=0xc001499258 sp=0xc001499210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc001423260, {0xc00139f800, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0014992b8 sp=0xc001499258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000dc1428, {0x5b0f6c0, 0xc001423260}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc001499320 sp=0xc0014992b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000dc1180, {0x5b0f840?, 0xc000640318}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001499390 sp=0xc001499320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000dc1180, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0014999c0 sp=0xc001499390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000dc1180, {0xc0014b0000, 0x8000, 0x4?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001499aa8 sp=0xc0014999c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0014922c0, {0xc0014b0000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001499af0 sp=0xc001499aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc0014ac540, {0xc000c498c0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001499bb0 sp=0xc001499af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0014ac540}, {0xc000c498c0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001499bf8 sp=0xc001499bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000c498c0, 0x9, 0x9}, {0x5b0f0a0, 0xc0014ac540}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001499c98 sp=0xc001499bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000c49880) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001499dd0 sp=0xc001499c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001128b40, 0xc0004efe00?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc001499fb0 sp=0xc001499dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc001499fe0 sp=0xc001499fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001499fe8 sp=0xc001499fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 376 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 581 [select]: +runtime.gopark(0xc00164c090?, 0x3?, 0x0?, 0x0?, 0xc001bf808a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001243808 sp=0xc0012437e8 pc=0x4777ae +runtime.selectgo(0xc00164c090, 0xc001bf8084, 0x0?, 0x0, 0xc00163e100?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001243958 sp=0xc001243808 pc=0x48916b +reflect.rselect({0xc001243b40, 0x3, 0x447c9e?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:589 +0x345 fp=0xc001243a08 sp=0xc001243958 pc=0x4a8f05 +reflect.Select({0xc0011d0540, 0x3, 0x2?}) + /home/ec2-user/sdk/go1.21.0/src/reflect/value.go:3104 +0xb8a fp=0xc001243bd0 sp=0xc001243a08 pc=0x57c44a +github.com/rancher/opni/pkg/util.WaitAll({0x5b426e0, 0xc0020885a0}, 0xc00163e090, {0xc001243db8, 0x2, 0x2}) + /home/ec2-user/opni/pkg/util/servers.go:51 +0x3ea fp=0xc001243d50 sp=0xc001243bd0 pc=0x176ae8a +github.com/rancher/opni/pkg/gateway.(*Gateway).ListenAndServe(0xc000912b40, {0x5b426e0, 0xc000b69f40}) + /home/ec2-user/opni/pkg/gateway/gateway.go:347 +0x2ae fp=0xc001243dd8 sp=0xc001243d50 pc=0x22b25ae +github.com/rancher/opni/pkg/test.(*Environment).startGateway.func3(0x4b0571) + /home/ec2-user/opni/pkg/test/environment.go:1709 +0x18c fp=0xc001243f88 sp=0xc001243dd8 pc=0x24c538c +github.com/rancher/opni/pkg/plugins/hooks.onLoadingCompletedHook.Invoke(0xc0007d3100, 0x7f429a42eb00?) + /home/ec2-user/opni/pkg/plugins/hooks/complete.go:10 +0x34 fp=0xc001243fa8 sp=0xc001243f88 pc=0x2182b94 +github.com/rancher/opni/pkg/plugins.(*PluginLoader).Complete.func1() + /home/ec2-user/opni/pkg/plugins/loader.go:329 +0x51 fp=0xc001243fe0 sp=0xc001243fa8 pc=0x2189711 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001243fe8 sp=0xc001243fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins.(*PluginLoader).Complete in goroutine 65 + /home/ec2-user/opni/pkg/plugins/loader.go:329 +0x138 + +goroutine 495 [select]: +runtime.gopark(0xc0011ffed0?, 0x2?, 0xb4?, 0x1b?, 0xc0011ffe4c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029c1c90 sp=0xc0029c1c70 pc=0x4777ae +runtime.selectgo(0xc0029c1ed0, 0xc0011ffe48, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029c1de0 sp=0xc0029c1c90 pc=0x48916b +github.com/rancher/opni/pkg/management.(*Server).listenAndServeGrpc(0xc000d8a000, {0x5b426e0, 0xc001152370}) + /home/ec2-user/opni/pkg/management/server.go:215 +0x3cd fp=0xc0029c1f20 sp=0xc0029c1de0 pc=0x231e52d +github.com/rancher/opni/pkg/management.(*Server).ListenAndServe.func1() + /home/ec2-user/opni/pkg/management/server.go:181 +0x4b fp=0xc0029c1fa0 sp=0xc0029c1f20 pc=0x231e0ab +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0029c1fe0 sp=0xc0029c1fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029c1fe8 sp=0xc0029c1fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 580 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 381 [select]: +runtime.gopark(0xc001433e70?, 0x2?, 0x0?, 0x0?, 0xc001433df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029c3c58 sp=0xc0029c3c38 pc=0x4777ae +runtime.selectgo(0xc0029c3e70, 0xc001433df0, 0xc001433e00?, 0x0, 0xc000dc1180?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029c3da8 sp=0xc0029c3c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00139d180, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0029c3ea0 sp=0xc0029c3da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0005e4460) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0029c3f58 sp=0xc0029c3ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc0029c3fe0 sp=0xc0029c3f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029c3fe8 sp=0xc0029c3fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 376 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 403 [select]: +runtime.gopark(0xc00119dfb0?, 0x2?, 0x26?, 0xaf?, 0xc00119dee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00119dd58 sp=0xc00119dd38 pc=0x4777ae +runtime.selectgo(0xc00119dfb0, 0xc00119dee0, 0xe?, 0x0, 0x4b0571?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00119dea8 sp=0xc00119dd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc00119dfe0 sp=0xc00119dea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00119dfe8 sp=0xc00119dfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 353 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 404 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0xe0?, 0x4d?, 0x22c5a46?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001114d60 sp=0xc001114d40 pc=0x4777ae +runtime.chanrecv(0xc001436360, 0xc001114e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001114dd8 sp=0xc001114d60 pc=0x441325 +runtime.chanrecv1(0xc00158a1c0?, 0xc00073a6b1?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001114e00 sp=0xc001114dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001114fe0 sp=0xc001114e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001114fe8 sp=0xc001114fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 353 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 410 [select]: +runtime.gopark(0xc001159f70?, 0x4?, 0x9?, 0x0?, 0xc001159dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001159c00 sp=0xc001159be0 pc=0x4777ae +runtime.selectgo(0xc001159f70, 0xc001159dd0, 0xc001159da8?, 0x0, 0xc000f8e380?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001159d50 sp=0xc001159c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00146c9c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc001159fc0 sp=0xc001159d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc001159fe0 sp=0xc001159fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001159fe8 sp=0xc001159fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 379 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 411 [IO wait]: +runtime.gopark(0xc00126cf88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00126cf40 sp=0xc00126cf20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00126cf78 sp=0xc00126cf40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677408, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00126cf98 sp=0xc00126cf78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000dbbf20, 0xc0014a0000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00126cfe0 sp=0xc00126cf98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000dbbf00, {0xc0014a0000, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00126d0d0 sp=0xc00126cfe0 pc=0x50e5a5 +net.(*netFD).Read(0xc000dbbf00, {0xc0014a0000, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00126d130 sp=0xc00126d0d0 pc=0x89342b +net.(*conn).Read(0xc000640320, {0xc0014a0000, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00126d1b8 sp=0xc00126d130 pc=0x8b176d +net.(*UnixConn).Read(0xc000640320, {0xc0014a0000, 0x240, 0x240}) + :1 +0x54 fp=0xc00126d200 sp=0xc00126d1b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc001423308, {0xc0014a0000, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc00126d260 sp=0xc00126d200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000dc17a8, {0x5b0f6c0, 0xc001423308}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc00126d2c8 sp=0xc00126d260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000dc1500, {0x5b0f840?, 0xc000640320}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc00126d338 sp=0xc00126d2c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000dc1500, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc00126d968 sp=0xc00126d338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000dc1500, {0xc0015be000, 0x8000, 0x4b0751?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc00126da50 sp=0xc00126d968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc00127d5a0, {0xc0015be000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc00126da98 sp=0xc00126da50 pc=0x10969ed +bufio.(*Reader).Read(0xc00127acc0, {0xc000f8e4a0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00126db58 sp=0xc00126da98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00127acc0}, {0xc000f8e4a0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00126dba0 sp=0xc00126db58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000f8e4a0, 0x9, 0x9}, {0x5b0f0a0, 0xc00127acc0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00126dc40 sp=0xc00126dba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000f8e460) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00126dd78 sp=0xc00126dc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00146c9c0, 0x4208e0?, 0x7f429a420400?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc00126dec8 sp=0xc00126dd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc0004efe00, {0x5b54e00?, 0xc00146c9c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc00126df80 sp=0xc00126dec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc00126dfe0 sp=0xc00126df80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00126dfe8 sp=0xc00126dfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 379 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 420 [select]: +runtime.gopark(0xc00115af28?, 0x2?, 0xb4?, 0x1b?, 0xc00115aebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00115ad30 sp=0xc00115ad10 pc=0x4777ae +runtime.selectgo(0xc00115af28, 0xc00115aeb8, 0x22c21f5?, 0x0, 0xc00115af20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00115ae80 sp=0xc00115ad30 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc0015de0c0, 0xc0014234d0, 0xc00158aba0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc00115af58 sp=0xc00115ae80 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeManagementAPI(0xc0015de0c0, {0x5b6f770?, 0xc000d8a000}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:185 +0x10a fp=0xc00115af98 sp=0xc00115af58 pc=0x21a7e0a +github.com/rancher/opni/pkg/management.NewServer.func1.2() + /home/ec2-user/opni/pkg/management/server.go:156 +0x62 fp=0xc00115afe0 sp=0xc00115af98 pc=0x231c0c2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00115afe8 sp=0xc00115afe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.NewServer.func1 in goroutine 417 + /home/ec2-user/opni/pkg/management/server.go:156 +0x137 + +goroutine 421 [select]: +runtime.gopark(0xc00115bf28?, 0x2?, 0xb4?, 0x1b?, 0xc00115bebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00115bd30 sp=0xc00115bd10 pc=0x4777ae +runtime.selectgo(0xc00115bf28, 0xc00115beb8, 0x1f?, 0x0, 0xc00115bf20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00115be80 sp=0xc00115bd30 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc0015de0c0, 0xc0014235a8, 0xc00158ac60) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc00115bf58 sp=0xc00115be80 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeNodeManagerServer(0xc0015de0c0, {0x5b21b70?, 0xc000ef5240}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:198 +0x10a fp=0xc00115bf98 sp=0xc00115bf58 pc=0x21a810a +github.com/rancher/opni/pkg/management.NewServer.func1.3() + /home/ec2-user/opni/pkg/management/server.go:158 +0x62 fp=0xc00115bfe0 sp=0xc00115bf98 pc=0x231c022 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00115bfe8 sp=0xc00115bfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.NewServer.func1 in goroutine 417 + /home/ec2-user/opni/pkg/management/server.go:158 +0x245 + +goroutine 649 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc0010e3930?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000c5ddd8 sp=0xc000c5ddb8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc001641e50, 0x3a) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc000c5de28 sp=0xc000c5ddd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc001641e40) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc000c5de58 sp=0xc000c5de28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc0020ace00, 0xc0010e3930) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc000c5dfb0 sp=0xc000c5de58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc000c5dfe0 sp=0xc000c5dfb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000c5dfe8 sp=0xc000c5dfe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 609 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 425 [select]: +runtime.gopark(0xc000f73448?, 0x2?, 0xd0?, 0x32?, 0xc000f733fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f73278 sp=0xc000f73258 pc=0x4777ae +runtime.selectgo(0xc000f73448, 0xc000f733f8, 0xc000813988?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000f733c8 sp=0xc000f73278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00146b560) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc000f73478 sp=0xc000f733c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001471d40, {0x42219c0?, 0xc0015de270}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc000f736d8 sp=0xc000f73478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00146b3d4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc000f73738 sp=0xc000f736d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00146b320, 0xc000f738f0, 0xc000f738c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc000f737e0 sp=0xc000f73738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00146b320, {0x42219c0, 0xc0015de270}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc000f73920 sp=0xc000f737e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc0015de330}, {0x4661da0, 0x1f}, {0x4278de0, 0xc0015de240}, {0x42219c0, 0xc0015de270}, 0x0?, {0xc000996a60, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc000f739c0 sp=0xc000f73920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc000762460}, {0x4661da0, 0x1f}, {0x4278de0, 0xc0015de240}, {0x42219c0, 0xc0015de270}, 0xc000f7a000, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc000f73d30 sp=0xc000f739c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc000f7a000, {0x5b426e0, 0xc000762460}, {0x4661da0, 0x1f}, {0x4278de0, 0xc0015de240}, {0x42219c0, 0xc0015de270}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc000f73e58 sp=0xc000f73d30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseManagementAPI(0xc00158aa70, {0x5b426e0, 0xc000762460}, 0x4b0a55?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:51 +0xee fp=0xc000f73ee8 sp=0xc000f73e58 pc=0x21ab66e +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeManagementAPI.func2(0x1) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:190 +0xf2 fp=0xc000f73f58 sp=0xc000f73ee8 pc=0x21a7f32 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc000f73fe0 sp=0xc000f73f58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f73fe8 sp=0xc000f73fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 420 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 426 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x60?, 0xe0?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001559cd0 sp=0xc001559cb0 pc=0x4777ae +runtime.chanrecv(0xc00127bda0, 0xc001559de0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001559d48 sp=0xc001559cd0 pc=0x441325 +runtime.chanrecv1(0xc001559df0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001559d70 sp=0xc001559d48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc001559f30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc001559e00 sp=0xc001559d70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc000ee4410, 0x2?, 0xc0015de510) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc001559fa0 sp=0xc001559e00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc001559fe0 sp=0xc001559fa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001559fe8 sp=0xc001559fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 421 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 427 [select]: +runtime.gopark(0xc001165448?, 0x2?, 0xd0?, 0x52?, 0xc0011653fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001165278 sp=0xc001165258 pc=0x4777ae +runtime.selectgo(0xc001165448, 0xc0011653f8, 0xc000813988?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011653c8 sp=0xc001165278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0015f8000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001165478 sp=0xc0011653c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc0015f6000, {0x42219c0?, 0xc0015deb40}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0011656d8 sp=0xc001165478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00146bdf4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001165738 sp=0xc0011656d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00146bd40, 0xc0011658f0, 0xc0011658c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc0011657e0 sp=0xc001165738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00146bd40, {0x42219c0, 0xc0015deb40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001165920 sp=0xc0011657e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc0015dec00}, {0x4673e80, 0x23}, {0x4278de0, 0xc0015deb10}, {0x42219c0, 0xc0015deb40}, 0x0?, {0xc000996a60, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc0011659c0 sp=0xc001165920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc000762460}, {0x4673e80, 0x23}, {0x4278de0, 0xc0015deb10}, {0x42219c0, 0xc0015deb40}, 0xc000f7a000, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc001165d30 sp=0xc0011659c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc000f7a000, {0x5b426e0, 0xc000762460}, {0x4673e80, 0x23}, {0x4278de0, 0xc0015deb10}, {0x42219c0, 0xc0015deb40}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc001165e58 sp=0xc001165d30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseNodeManagerClient(0xc00158aa70, {0x5b426e0, 0xc000762460}, 0x4b0a55?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:60 +0xee fp=0xc001165ee8 sp=0xc001165e58 pc=0x21ab7ee +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeNodeManagerServer.func2(0x2) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:203 +0xf2 fp=0xc001165f58 sp=0xc001165ee8 pc=0x21a8232 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc001165fe0 sp=0xc001165f58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001165fe8 sp=0xc001165fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 421 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 430 [select]: +runtime.gopark(0xc000d38f38?, 0x2?, 0x69?, 0x97?, 0xc000d38f14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d38d90 sp=0xc000d38d70 pc=0x4777ae +runtime.selectgo(0xc000d38f38, 0xc000d38f10, 0xc000d38f18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d38ee0 sp=0xc000d38d90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc000d38f68 sp=0xc000d38ee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc0014238d8?, 0xc00158b000?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc000d38fa8 sp=0xc000d38f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc000d38fe0 sp=0xc000d38fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d38fe8 sp=0xc000d38fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 424 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 431 [chan receive]: +runtime.gopark(0x0?, 0x2?, 0x1?, 0x26?, 0xff000000?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00148aef8 sp=0xc00148aed8 pc=0x4777ae +runtime.chanrecv(0xc0010886c0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00148af70 sp=0xc00148aef8 pc=0x441325 +runtime.chanrecv1(0xc000762460?, 0x2106c90?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00148af98 sp=0xc00148af70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc00148afe0 sp=0xc00148af98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00148afe8 sp=0xc00148afe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 426 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 432 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015ef978 sp=0xc0015ef958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0015ef9b0 sp=0xc0015ef978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677218, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0015ef9d0 sp=0xc0015ef9b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00144f1a0, 0xc0015efa01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0015efa18 sp=0xc0015ef9d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc00144f180) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0015efb08 sp=0xc0015efa18 pc=0x513f65 +net.(*netFD).accept(0xc00144f180) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0015efc00 sp=0xc0015efb08 pc=0x8967be +net.(*UnixListener).accept(0xc0015deae0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc0015efc40 sp=0xc0015efc00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc0015deae0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc0015efcb0 sp=0xc0015efc40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc00127dbe0) + :1 +0x44 fp=0xc0015efce8 sp=0xc0015efcb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc00141a5a0, {0x5b2c6a0, 0xc00127dbe0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc0015eff18 sp=0xc0015efce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc0015eff68 sp=0xc0015eff18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc00127dfe0?, 0xc00158b130?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc0015effa8 sp=0xc0015eff68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc0015effe0 sp=0xc0015effa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015effe8 sp=0xc0015effe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 426 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 433 [select]: +runtime.gopark(0xc00119ef38?, 0x2?, 0x0?, 0x0?, 0xc00119ef14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00119ed90 sp=0xc00119ed70 pc=0x4777ae +runtime.selectgo(0xc00119ef38, 0xc00119ef10, 0xc00119ef18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00119eee0 sp=0xc00119ed90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc00119ef68 sp=0xc00119eee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc001423998?, 0xc00158b150?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc00119efa8 sp=0xc00119ef68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc00119efe0 sp=0xc00119efa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00119efe8 sp=0xc00119efe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 426 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 434 [chan receive]: +runtime.gopark(0xc0007d32a4?, 0x4be6c0?, 0xa0?, 0x32?, 0x4be720?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012d0cf8 sp=0xc0012d0cd8 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0012d0d70 sp=0xc0012d0cf8 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0xc001bbbe30?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0012d0d98 sp=0xc0012d0d70 pc=0x440f72 +github.com/rancher/opni/plugins/metrics/pkg/gateway.(*Plugin).UseManagementAPI(0xc000cad180, {0x5b6f168, 0xc001661580}) + /home/ec2-user/opni/plugins/metrics/pkg/gateway/system.go:69 +0x67c fp=0xc0012d0fa0 sp=0xc0012d0d98 pc=0x3db3f9c +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseManagementAPI(0xc000e75840, {0x4b05a9?, 0xc00126f0f0?}, 0xc001560180) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:102 +0x30f fp=0xc0012d1098 sp=0xc0012d0fa0 pc=0x21a640f +github.com/rancher/opni/pkg/plugins/apis/system._System_UseManagementAPI_Handler.func1({0x5b426a8, 0xc001560240}, {0x4278de0?, 0xc001560180}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:151 +0x83 fp=0xc0012d10f0 sp=0xc0012d1098 pc=0x21ac063 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc001560240}, {0x4278de0, 0xc001560180}, 0xc001331520, 0xc001321068) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc0012d1148 sp=0xc0012d10f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc001560240}, {0x4278de0, 0xc001560180}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc0012d11e8 sp=0xc0012d1148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc001560150}, {0x4278de0, 0xc001560180}, 0xc001331520, 0xc001367f40) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc0012d16b8 sp=0xc0012d11e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc001560150}, {0x4278de0, 0xc001560180}, 0xc00126f778?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc0012d1740 sp=0xc0012d16b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseManagementAPI_Handler({0x4278ea0?, 0xc000e75840}, {0x5b426a8, 0xc001560150}, 0xc000611420, 0xc000b74c80) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:153 +0x1c4 fp=0xc0012d17d0 sp=0xc0012d1740 pc=0x21abec4 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc0015f8120, 0xc000c4af00, 0x771eca0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc0012d1db0 sp=0xc0012d17d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc0015f8120, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc0012d1f28 sp=0xc0012d1db0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0012d1fe0 sp=0xc0012d1f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012d1fe8 sp=0xc0012d1fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1136 [select]: +runtime.gopark(0xc002f89478?, 0x2?, 0x80?, 0x93?, 0xc002f89414?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002f89280 sp=0xc002f89260 pc=0x4777ae +runtime.selectgo(0xc002f89478, 0xc002f89410, 0xc0020868a0?, 0x0, 0xc002333470?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002f893d0 sp=0xc002f89280 pc=0x48916b +github.com/rancher/opni/pkg/management.(*Server).WatchClusterHealthStatus(0xc0010c0e70, 0x42219c0?, {0x5b4cae0, 0xc001bf00d0}) + /home/ec2-user/opni/pkg/management/health.go:38 +0x352 fp=0xc002f894a8 sp=0xc002f893d0 pc=0x23189f2 +github.com/rancher/opni/pkg/apis/management/v1._Management_WatchClusterHealthStatus_Handler({0x4537040?, 0xc0010c0e70}, {0x5b4ab30, 0xc002086960}) + /home/ec2-user/opni/pkg/apis/management/v1/management_grpc.pb.go:829 +0xfa fp=0xc002f89508 sp=0xc002f894a8 pc=0x1781ada +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4537040, 0xc0010c0e70}, {0x5b492f8, 0xc001da4000}, 0xc0028fd2f0, 0x47ec6b8) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002f89848 sp=0xc002f89508 pc=0x12b0e02 +google.golang.org/grpc.(*Server).processStreamingRPC(0xc001ebfc20, {0x5b54e00, 0xc0021d44e0}, 0xc0015f98c0, 0xc001be2420, 0x7716c80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002f89db0 sp=0xc002f89848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc001ebfc20, {0x5b54e00, 0xc0021d44e0}, 0xc0015f98c0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002f89f28 sp=0xc002f89db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002f89fe0 sp=0xc002f89f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002f89fe8 sp=0xc002f89fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1007 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 438 [select]: +runtime.gopark(0xc001185f70?, 0x2?, 0xb4?, 0x1b?, 0xc001185ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001185d20 sp=0xc001185d00 pc=0x4777ae +runtime.selectgo(0xc001185f70, 0xc001185eb8, 0xc001175040?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001185e70 sp=0xc001185d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0015fe360, {0x5b426e0, 0xc0015fa730}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001185fa0 sp=0xc001185e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001185fe0 sp=0xc001185fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001185fe8 sp=0xc001185fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 435 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 437 [chan receive]: +runtime.gopark(0x4be6c0?, 0xc000f1df50?, 0x20?, 0xe7?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001726f08 sp=0xc001726ee8 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001726f80 sp=0xc001726f08 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0x5b11a20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001726fa8 sp=0xc001726f80 pc=0x440f72 +github.com/rancher/opni/plugins/metrics/pkg/gateway.(*Plugin).UseNodeManagerClient(0xc000cad180, {0x5b11a20, 0xc0014d6a50}) + /home/ec2-user/opni/plugins/metrics/pkg/gateway/system.go:74 +0x93 fp=0xc001726fd0 sp=0xc001726fa8 pc=0x3db4593 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseNodeManagerClient(0xc000e75840, {0x4b05a9?, 0xc0017190f0?}, 0xc0015dfe00) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:113 +0x18d fp=0xc001727098 sp=0xc001726fd0 pc=0x21a68cd +github.com/rancher/opni/pkg/plugins/apis/system._System_UseNodeManagerClient_Handler.func1({0x5b426a8, 0xc0015dfec0}, {0x4278de0?, 0xc0015dfe00}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:169 +0x83 fp=0xc0017270f0 sp=0xc001727098 pc=0x21ac443 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc0015dfec0}, {0x4278de0, 0xc0015dfe00}, 0xc0015fe460, 0xc001423de8) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc001727148 sp=0xc0017270f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc0015dfec0}, {0x4278de0, 0xc0015dfe00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc0017271e8 sp=0xc001727148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc0015dfdd0}, {0x4278de0, 0xc0015dfe00}, 0xc0015fe460, 0xc0015d9880) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc0017276b8 sp=0xc0017271e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc0015dfdd0}, {0x4278de0, 0xc0015dfe00}, 0xc001719778?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc001727740 sp=0xc0017276b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseNodeManagerClient_Handler({0x4278ea0?, 0xc000e75840}, {0x5b426a8, 0xc0015dfdd0}, 0xc000611500, 0xc000b74c80) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:171 +0x1c4 fp=0xc0017277d0 sp=0xc001727740 pc=0x21ac2a4 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc0015f8480, 0xc000c4af00, 0x771ecb8, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc001727db0 sp=0xc0017277d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc0015f8480, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc001727f28 sp=0xc001727db0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001727fe0 sp=0xc001727f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001727fe8 sp=0xc001727fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 439 [select]: +runtime.gopark(0xc0015f1f70?, 0x2?, 0xb4?, 0x1b?, 0xc0015f1ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015f1d20 sp=0xc0015f1d00 pc=0x4777ae +runtime.selectgo(0xc0015f1f70, 0xc0015f1eb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0015f1e70 sp=0xc0015f1d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0015fe3c0, {0x5b426e0, 0xc0015fa780}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0015f1fa0 sp=0xc0015f1e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0015f1fe0 sp=0xc0015f1fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015f1fe8 sp=0xc0015f1fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 435 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 625 [select]: +runtime.gopark(0xc002ed9e40?, 0x3?, 0xb4?, 0x1b?, 0xc002ed9c2a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ed9a48 sp=0xc002ed9a28 pc=0x4777ae +runtime.selectgo(0xc002ed9e40, 0xc002ed9c24, 0xc002bfaf0b?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ed9b98 sp=0xc002ed9a48 pc=0x48916b +github.com/rancher/opni/pkg/health.(*Monitor).Run(0xc000ee4050, {0x5b426e0, 0xc001fee820}, {0x5b21aa8, 0xc001623400}) + /home/ec2-user/opni/pkg/health/monitor.go:69 +0x1f3 fp=0xc002ed9f80 sp=0xc002ed9b98 pc=0x206e173 +github.com/rancher/opni/pkg/gateway.NewGateway.func23() + /home/ec2-user/opni/pkg/gateway/gateway.go:277 +0x6c fp=0xc002ed9fe0 sp=0xc002ed9f80 pc=0x22aed8c +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ed9fe8 sp=0xc002ed9fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.NewGateway in goroutine 609 + /home/ec2-user/opni/pkg/gateway/gateway.go:277 +0x3eed + +goroutine 401 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x26?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a50bd0 sp=0xc000a50bb0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000a50c08 sp=0xc000a50bd0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a968, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000a50c28 sp=0xc000a50c08 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001b16ba0, 0xc001b5e000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000a50c70 sp=0xc000a50c28 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001b16b80, {0xc001b5e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc000a50d60 sp=0xc000a50c70 pc=0x50e5a5 +net.(*netFD).Read(0xc001b16b80, {0xc001b5e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc000a50dc0 sp=0xc000a50d60 pc=0x89342b +net.(*conn).Read(0xc000bde888, {0xc001b5e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc000a50e48 sp=0xc000a50dc0 pc=0x8b176d +net.(*TCPConn).Read(0xc000bde888, {0xc001b5e000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc000a50e90 sp=0xc000a50e48 pc=0x8d72f4 +github.com/nats-io/nats%2ego.(*natsReader).Read(0xc0012b5500) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1890 +0x1a9 fp=0xc000a50f10 sp=0xc000a50e90 pc=0x1f6d169 +github.com/nats-io/nats%2ego.(*Conn).readLoop(0xc001657c00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2917 +0x1f8 fp=0xc000a50fc0 sp=0xc000a50f10 pc=0x1f75fb8 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x34 fp=0xc000a50fe0 sp=0xc000a50fc0 pc=0x1f71514 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a50fe8 sp=0xc000a50fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 434 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x44e + +goroutine 485 [select]: +runtime.gopark(0xc0014a9e48?, 0x2?, 0x0?, 0x0?, 0xc0014a9dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029cdc30 sp=0xc0029cdc10 pc=0x4777ae +runtime.selectgo(0xc0029cde48, 0xc0014a9dc8, 0xc0014a9dd8?, 0x0, 0xc000dc0a80?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029cdd80 sp=0xc0029cdc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00139d860, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0029cde78 sp=0xc0029cdd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000611a40) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0029cdf30 sp=0xc0029cde78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc0029cdfe0 sp=0xc0029cdf30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029cdfe8 sp=0xc0029cdfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 453 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 675 [select]: +runtime.gopark(0xc0017c9e48?, 0x2?, 0x0?, 0x0?, 0xc0017c9dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001127c30 sp=0xc001127c10 pc=0x4777ae +runtime.selectgo(0xc001127e48, 0xc0017c9dc8, 0xc0017c9dd8?, 0x0, 0xc001d6f220?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001127d80 sp=0xc001127c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00099bea0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001127e78 sp=0xc001127d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001d8ce00) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001127f30 sp=0xc001127e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc001127fe0 sp=0xc001127f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001127fe8 sp=0xc001127fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 674 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 496 [select]: +runtime.gopark(0xc00151ded0?, 0x2?, 0xb4?, 0x1b?, 0xc00151de1c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029cfc68 sp=0xc0029cfc48 pc=0x4777ae +runtime.selectgo(0xc0029cfed0, 0xc00151de18, 0x1?, 0x0, 0xc001622480?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029cfdb8 sp=0xc0029cfc68 pc=0x48916b +github.com/rancher/opni/pkg/management.(*Server).listenAndServeHttp(0xc000d8a000, {0x5b426e0?, 0xc001152370}) + /home/ec2-user/opni/pkg/management/server.go:248 +0x625 fp=0xc0029cff20 sp=0xc0029cfdb8 pc=0x231ed05 +github.com/rancher/opni/pkg/management.(*Server).ListenAndServe.func2() + /home/ec2-user/opni/pkg/management/server.go:189 +0x4b fp=0xc0029cffa0 sp=0xc0029cff20 pc=0x231dfab +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0029cffe0 sp=0xc0029cffa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029cffe8 sp=0xc0029cffe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 580 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 390 [select]: +runtime.gopark(0xc001715f70?, 0x2?, 0xb4?, 0x1b?, 0xc001715ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001715d20 sp=0xc001715d00 pc=0x4777ae +runtime.selectgo(0xc001715f70, 0xc001715eb8, 0x2322150?, 0x0, 0x23222f5?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001715e70 sp=0xc001715d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001634980, {0x5b426e0, 0xc0016620a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001715fa0 sp=0xc001715e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001715fe0 sp=0xc001715fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001715fe8 sp=0xc001715fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 437 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 391 [select]: +runtime.gopark(0xc0010eff70?, 0x2?, 0xb4?, 0x1b?, 0xc0010efebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010efd20 sp=0xc0010efd00 pc=0x4777ae +runtime.selectgo(0xc0010eff70, 0xc0010efeb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010efe70 sp=0xc0010efd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0016349e0, {0x5b426e0, 0xc0016620f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0010effa0 sp=0xc0010efe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0010effe0 sp=0xc0010effa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010effe8 sp=0xc0010effe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 437 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 319 [select]: +runtime.gopark(0xc001719f70?, 0x2?, 0xb4?, 0x1b?, 0xc001719ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001719d20 sp=0xc001719d00 pc=0x4777ae +runtime.selectgo(0xc001719f70, 0xc001719eb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001719e70 sp=0xc001719d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001331720, {0x5b426e0, 0xc00137d400}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001719fa0 sp=0xc001719e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001719fe0 sp=0xc001719fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001719fe8 sp=0xc001719fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 434 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 320 [select]: +runtime.gopark(0xc00126ff70?, 0x2?, 0xb4?, 0x1b?, 0xc00126febc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00126fd20 sp=0xc00126fd00 pc=0x4777ae +runtime.selectgo(0xc00126ff70, 0xc00126feb8, 0x472ddb?, 0x0, 0x4ad421?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00126fe70 sp=0xc00126fd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001331780, {0x5b426e0, 0xc00137d450}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc00126ffa0 sp=0xc00126fe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc00126ffe0 sp=0xc00126ffa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00126ffe8 sp=0xc00126ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 434 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1120 [select]: +runtime.gopark(0xc0029bbf50?, 0x2?, 0xb4?, 0x1b?, 0xc0029bbeac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029bbce8 sp=0xc0029bbcc8 pc=0x4777ae +runtime.selectgo(0xc0029bbf50, 0xc0029bbea8, 0xc001305b00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029bbe38 sp=0xc0029bbce8 pc=0x48916b +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).handleDriverNotifications(0xc00070a000) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:227 +0x125 fp=0xc0029bbfc0 sp=0xc0029bbe38 pc=0x3972405 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI.func2() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:89 +0x34 fp=0xc0029bbfe0 sp=0xc0029bbfc0 pc=0x396f0f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029bbfe8 sp=0xc0029bbfe0 pc=0x4ad421 +created by github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI in goroutine 886 + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:89 +0x54b + +goroutine 546 [chan receive]: +runtime.gopark(0x4aef8c?, 0x4b0a55?, 0x28?, 0x7c?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015e8e18 sp=0xc0015e8df8 pc=0x4777ae +runtime.chanrecv(0xc001089da0, 0xc0015e8ed0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0015e8e90 sp=0xc0015e8e18 pc=0x441325 +runtime.chanrecv2(0xc001657c28?, 0x11148f1?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0015e8eb8 sp=0xc0015e8e90 pc=0x440f92 +github.com/nats-io/nats%2ego.(*Conn).flusher(0xc001657c00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:3324 +0x1a5 fp=0xc0015e8fc0 sp=0xc0015e8eb8 pc=0x1f792e5 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x34 fp=0xc0015e8fe0 sp=0xc0015e8fc0 pc=0x1f714b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015e8fe8 sp=0xc0015e8fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 434 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x4cb + +goroutine 454 [IO wait]: +runtime.gopark(0xc00171afe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00171af98 sp=0xc00171af78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00171afd0 sp=0xc00171af98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677120, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00171aff0 sp=0xc00171afd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0014d33a0, 0xc001941800?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00171b038 sp=0xc00171aff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0014d3380, {0xc001941800, 0x3800, 0x3800}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00171b128 sp=0xc00171b038 pc=0x50e5a5 +net.(*netFD).Read(0xc0014d3380, {0xc001941800, 0x3800, 0x3800}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00171b188 sp=0xc00171b128 pc=0x89342b +net.(*conn).Read(0xc000640000, {0xc001941800, 0x3800, 0x3800}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00171b210 sp=0xc00171b188 pc=0x8b176d +net.(*UnixConn).Read(0xc000640000, {0xc001941800, 0x3800, 0x3800}) + :1 +0x54 fp=0xc00171b258 sp=0xc00171b210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0004a7ce0, {0xc001941800, 0x3800, 0x3800}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc00171b2b8 sp=0xc00171b258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000dc0628, {0x5b0f6c0, 0xc0004a7ce0}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc00171b320 sp=0xc00171b2b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000dc0380, {0x5b0f840?, 0xc000640000}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc00171b390 sp=0xc00171b320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000dc0380, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc00171b9c0 sp=0xc00171b390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000dc0380, {0xc001010000, 0x8000, 0xc00106e550?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc00171baa8 sp=0xc00171b9c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001492980, {0xc001010000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc00171baf0 sp=0xc00171baa8 pc=0x10969ed +bufio.(*Reader).Read(0xc000f1a7e0, {0xc000886040, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00171bbb0 sp=0xc00171baf0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000f1a7e0}, {0xc000886040, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00171bbf8 sp=0xc00171bbb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000886040, 0x9, 0x9}, {0x5b0f0a0, 0xc000f1a7e0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00171bc98 sp=0xc00171bbf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000886000) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00171bdd0 sp=0xc00171bc98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001128900, 0xc00141a3c0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc00171bfb0 sp=0xc00171bdd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc00171bfe0 sp=0xc00171bfb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00171bfe8 sp=0xc00171bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 321 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 547 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc0007d3500?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bc1f08 sp=0xc001bc1ee8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc0012b54d0, 0x0) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc001bc1f58 sp=0xc001bc1f08 pc=0x4a9339 +sync.(*Cond).Wait(0xc0012b54c0) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc001bc1f88 sp=0xc001bc1f58 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*asyncCallbacksHandler).asyncCBDispatcher(0xc0007d3500) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2842 +0x153 fp=0xc001bc1fc0 sp=0xc001bc1f88 pc=0x1f759b3 +github.com/nats-io/nats%2ego.Options.Connect.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x34 fp=0xc001bc1fe0 sp=0xc001bc1fc0 pc=0x1f6a534 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bc1fe8 sp=0xc001bc1fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.Options.Connect in goroutine 434 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x86b + +goroutine 478 [IO wait]: +runtime.gopark(0xc001716fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001716f98 sp=0xc001716f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001716fd0 sp=0xc001716f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677028, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001716ff0 sp=0xc001716fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00166c5a0, 0xc000f14600?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001717038 sp=0xc001716ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00166c580, {0xc000f14600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001717128 sp=0xc001717038 pc=0x50e5a5 +net.(*netFD).Read(0xc00166c580, {0xc000f14600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001717188 sp=0xc001717128 pc=0x89342b +net.(*conn).Read(0xc0012c6000, {0xc000f14600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001717210 sp=0xc001717188 pc=0x8b176d +net.(*UnixConn).Read(0xc0012c6000, {0xc000f14600, 0x600, 0x600}) + :1 +0x54 fp=0xc001717258 sp=0xc001717210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc001320bb8, {0xc000f14600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0017172b8 sp=0xc001717258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0010b82a8, {0x5b0f6c0, 0xc001320bb8}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc001717320 sp=0xc0017172b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc0010b8000, {0x5b0f840?, 0xc0012c6000}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001717390 sp=0xc001717320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc0010b8000, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0017179c0 sp=0xc001717390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc0010b8000, {0xc00175e000, 0x8000, 0x4?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001717aa8 sp=0xc0017179c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001635040, {0xc00175e000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001717af0 sp=0xc001717aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc000c4ede0, {0xc000e4ec80, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001717bb0 sp=0xc001717af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000c4ede0}, {0xc000e4ec80, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001717bf8 sp=0xc001717bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000e4ec80, 0x9, 0x9}, {0x5b0f0a0, 0xc000c4ede0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001717c98 sp=0xc001717bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000e4ec40) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001717dd0 sp=0xc001717c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00165a6c0, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc001717fb0 sp=0xc001717dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc001717fe0 sp=0xc001717fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001717fe8 sp=0xc001717fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 392 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 610 [select]: +runtime.gopark(0xc000c5ff70?, 0x2?, 0xb4?, 0x1b?, 0xc000c5febc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000c5fd20 sp=0xc000c5fd00 pc=0x4777ae +runtime.selectgo(0xc000c5ff70, 0xc000c5feb8, 0xc0015fd520?, 0x0, 0x440940?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000c5fe70 sp=0xc000c5fd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001dec3a0, {0x5b426e0, 0xc001152640}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc000c5ffa0 sp=0xc000c5fe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc000c5ffe0 sp=0xc000c5ffa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000c5ffe8 sp=0xc000c5ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 496 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 455 [select]: +runtime.gopark(0xc001633e70?, 0x2?, 0x0?, 0x0?, 0xc001633df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029c9c58 sp=0xc0029c9c38 pc=0x4777ae +runtime.selectgo(0xc0029c9e70, 0xc001633df0, 0xc001633e00?, 0x0, 0xc000dc0380?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029c9da8 sp=0xc0029c9c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001662960, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0029c9ea0 sp=0xc0029c9da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004fe310) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0029c9f58 sp=0xc0029c9ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc0029c9fe0 sp=0xc0029c9f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029c9fe8 sp=0xc0029c9fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 321 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1387 [IO wait]: +runtime.gopark(0xc0024d2f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024d2f40 sp=0xc0024d2f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0024d2f78 sp=0xc0024d2f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c48a8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0024d2f98 sp=0xc0024d2f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00257d920, 0xc001156f00?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0024d2fe0 sp=0xc0024d2f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00257d900, {0xc001156f00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0024d30d0 sp=0xc0024d2fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc00257d900, {0xc001156f00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0024d3130 sp=0xc0024d30d0 pc=0x89342b +net.(*conn).Read(0xc001d6f070, {0xc001156f00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0024d31b8 sp=0xc0024d3130 pc=0x8b176d +net.(*UnixConn).Read(0xc001d6f070, {0xc001156f00, 0x500, 0x500}) + :1 +0x54 fp=0xc0024d3200 sp=0xc0024d31b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc00304a2e8, {0xc001156f00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0024d3260 sp=0xc0024d3200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc00133b428, {0x5b0f6c0, 0xc00304a2e8}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0024d32c8 sp=0xc0024d3260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc00133b180, {0x5b0f840?, 0xc001d6f070}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc0024d3338 sp=0xc0024d32c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc00133b180, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0024d3968 sp=0xc0024d3338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc00133b180, {0xc00282e000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc0024d3a50 sp=0xc0024d3968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001d86e60, {0xc00282e000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc0024d3a98 sp=0xc0024d3a50 pc=0x10969ed +bufio.(*Reader).Read(0xc002175980, {0xc0014f90e0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0024d3b58 sp=0xc0024d3a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc002175980}, {0xc0014f90e0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0024d3ba0 sp=0xc0024d3b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0014f90e0, 0x9, 0x9}, {0x5b0f0a0, 0xc002175980}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0024d3c40 sp=0xc0024d3ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0014f90a0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0024d3d78 sp=0xc0024d3c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0024e7860, 0xc001c6a0f0?, 0x5b2ad00?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc0024d3ec8 sp=0xc0024d3d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc0018534a0, {0x5b54e00?, 0xc0024e7860}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc0024d3f80 sp=0xc0024d3ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc0024d3fe0 sp=0xc0024d3f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024d3fe8 sp=0xc0024d3fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1394 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 396 [select]: +runtime.gopark(0xc0015afe48?, 0x2?, 0x0?, 0x0?, 0xc0015afdcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029cbc30 sp=0xc0029cbc10 pc=0x4777ae +runtime.selectgo(0xc0029cbe48, 0xc0015afdc8, 0xc0015afdd8?, 0x0, 0xc000dc0700?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029cbd80 sp=0xc0029cbc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00137dc70, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0029cbe78 sp=0xc0029cbd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0006e6070) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0029cbf30 sp=0xc0029cbe78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc0029cbfe0 sp=0xc0029cbf30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029cbfe8 sp=0xc0029cbfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 452 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 397 [select]: +runtime.gopark(0xc0015ebf70?, 0x4?, 0x38?, 0xdc?, 0xc0015ebdd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015ebc00 sp=0xc0015ebbe0 pc=0x4777ae +runtime.selectgo(0xc0015ebf70, 0xc0015ebdd0, 0x40a16a0?, 0x0, 0xc000e9ca20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0015ebd50 sp=0xc0015ebc00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000c7c000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc0015ebfc0 sp=0xc0015ebd50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc0015ebfe0 sp=0xc0015ebfc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015ebfe8 sp=0xc0015ebfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 452 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 398 [IO wait]: +runtime.gopark(0xc001534f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001534f40 sp=0xc001534f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001534f78 sp=0xc001534f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162ac50, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001534f98 sp=0xc001534f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0014d36a0, 0xc0014a0900?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001534fe0 sp=0xc001534f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0014d3680, {0xc0014a0900, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0015350d0 sp=0xc001534fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc0014d3680, {0xc0014a0900, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001535130 sp=0xc0015350d0 pc=0x89342b +net.(*conn).Read(0xc000640010, {0xc0014a0900, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0015351b8 sp=0xc001535130 pc=0x8b176d +net.(*UnixConn).Read(0xc000640010, {0xc0014a0900, 0x240, 0x240}) + :1 +0x54 fp=0xc001535200 sp=0xc0015351b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc001320bd0, {0xc0014a0900, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001535260 sp=0xc001535200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000dc09a8, {0x5b0f6c0, 0xc001320bd0}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0015352c8 sp=0xc001535260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000dc0700, {0x5b0f840?, 0xc000640010}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001535338 sp=0xc0015352c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000dc0700, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001535968 sp=0xc001535338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000dc0700, {0xc001456000, 0x8000, 0x0?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001535a50 sp=0xc001535968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001330ac0, {0xc001456000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001535a98 sp=0xc001535a50 pc=0x10969ed +bufio.(*Reader).Read(0xc0012494a0, {0xc000d5c040, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001535b58 sp=0xc001535a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0012494a0}, {0xc000d5c040, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001535ba0 sp=0xc001535b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000d5c040, 0x9, 0x9}, {0x5b0f0a0, 0xc0012494a0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001535c40 sp=0xc001535ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000d5c000) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001535d78 sp=0xc001535c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000c7c000, 0x0?, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001535ec8 sp=0xc001535d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00141a5a0, {0x5b54e00?, 0xc000c7c000}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001535f80 sp=0xc001535ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001535fe0 sp=0xc001535f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001535fe8 sp=0xc001535fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 452 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 456 [select]: +runtime.gopark(0xc001486f20?, 0x2?, 0xb4?, 0x1b?, 0xc001486eb4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001486d28 sp=0xc001486d08 pc=0x4777ae +runtime.selectgo(0xc001486f20, 0xc001486eb0, 0x0?, 0x0, 0xc001486f18?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001486e78 sp=0xc001486d28 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc0015de0c0, 0xc000fe1bd0, 0xc000fe1be0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001486f50 sp=0xc001486e78 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeKeyValueStore(0xc0015de0c0, {0x5b43370?, 0xc000fe1bc0}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:214 +0x1a8 fp=0xc001486f98 sp=0xc001486f50 pc=0x21a84a8 +github.com/rancher/opni/pkg/gateway.NewGateway.func2.1() + /home/ec2-user/opni/pkg/gateway/gateway.go:157 +0x62 fp=0xc001486fe0 sp=0xc001486f98 pc=0x22b18c2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001486fe8 sp=0xc001486fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.NewGateway.func2 in goroutine 419 + /home/ec2-user/opni/pkg/gateway/gateway.go:157 +0x3cc + +goroutine 459 [chan receive]: +runtime.gopark(0x78?, 0x7769700?, 0x70?, 0x5f?, 0x11782b6?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00148cef8 sp=0xc00148ced8 pc=0x4777ae +runtime.chanrecv(0xc0010886c0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00148cf70 sp=0xc00148cef8 pc=0x441325 +runtime.chanrecv1(0xc000762460?, 0x5b43370?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00148cf98 sp=0xc00148cf70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc00148cfe0 sp=0xc00148cf98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00148cfe8 sp=0xc00148cfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 457 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 460 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00149b978 sp=0xc00149b958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00149b9b0 sp=0xc00149b978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6e818, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00149b9d0 sp=0xc00149b9b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0014d3f20, 0xc00149ba01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00149ba18 sp=0xc00149b9d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0014d3f00) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc00149bb08 sp=0xc00149ba18 pc=0x513f65 +net.(*netFD).accept(0xc0014d3f00) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc00149bc00 sp=0xc00149bb08 pc=0x8967be +net.(*UnixListener).accept(0xc000ff9080) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc00149bc40 sp=0xc00149bc00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc000ff9080) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc00149bcb0 sp=0xc00149bc40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc001492dc0) + :1 +0x44 fp=0xc00149bce8 sp=0xc00149bcb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc0010a4d20, {0x5b2c6a0, 0xc001492dc0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc00149bf18 sp=0xc00149bce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc00149bf68 sp=0xc00149bf18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc001492f20?, 0xc000fe1f40?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc00149bfa8 sp=0xc00149bf68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc00149bfe0 sp=0xc00149bfa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00149bfe8 sp=0xc00149bfe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 457 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 461 [select]: +runtime.gopark(0xc0011daf38?, 0x2?, 0x0?, 0x0?, 0xc0011daf14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011dad90 sp=0xc0011dad70 pc=0x4777ae +runtime.selectgo(0xc0011daf38, 0xc0011daf10, 0xc0011daf18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011daee0 sp=0xc0011dad90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc0011daf68 sp=0xc0011daee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc00149e990?, 0xc000fe1f70?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc0011dafa8 sp=0xc0011daf68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc0011dafe0 sp=0xc0011dafa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011dafe8 sp=0xc0011dafe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 457 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 462 [chan receive]: +runtime.gopark(0xc0007d3304?, 0x4be6c0?, 0x0?, 0x33?, 0x4be720?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000bacdd0 sp=0xc000bacdb0 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc000bace48 sp=0xc000bacdd0 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0xc000ee6320?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc000bace70 sp=0xc000bace48 pc=0x440f72 +github.com/rancher/opni/plugins/metrics/pkg/gateway.(*Plugin).UseKeyValueStore(0xc000cad180, {0x5b43f90?, 0xc0018be840}) + /home/ec2-user/opni/plugins/metrics/pkg/gateway/system.go:91 +0x610 fp=0xc000bacfd0 sp=0xc000bace70 pc=0x3db4bf0 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseKeyValueStore(0xc000e75840, {0x4b05a9?, 0xc0015310f0?}, 0xc001150810) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:123 +0x18d fp=0xc000bad098 sp=0xc000bacfd0 pc=0x21a6c2d +github.com/rancher/opni/pkg/plugins/apis/system._System_UseKeyValueStore_Handler.func1({0x5b426a8, 0xc001150930}, {0x4278de0?, 0xc001150810}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:187 +0x83 fp=0xc000bad0f0 sp=0xc000bad098 pc=0x21ac823 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc001150930}, {0x4278de0, 0xc001150810}, 0xc001493360, 0xc00149ed38) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc000bad148 sp=0xc000bad0f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc001150930}, {0x4278de0, 0xc001150810}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc000bad1e8 sp=0xc000bad148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc0011507e0}, {0x4278de0, 0xc001150810}, 0xc001493360, 0xc0014d9900) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc000bad6b8 sp=0xc000bad1e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc0011507e0}, {0x4278de0, 0xc001150810}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc000bad740 sp=0xc000bad6b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseKeyValueStore_Handler({0x4278ea0?, 0xc000e75840}, {0x5b426a8, 0xc0011507e0}, 0xc0007d1ea0, 0xc000b74c80) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:189 +0x1c4 fp=0xc000bad7d0 sp=0xc000bad740 pc=0x21ac684 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc0010dcd80, 0xc000c4af00, 0x771ecd0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc000baddb0 sp=0xc000bad7d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc0010dcd80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc000badf28 sp=0xc000baddb0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc000badfe0 sp=0xc000badf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000badfe8 sp=0xc000badfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 600 [select]: +runtime.gopark(0xc000d06ed8?, 0x2?, 0xb4?, 0x1b?, 0xc000d06e84?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d06cf8 sp=0xc000d06cd8 pc=0x4777ae +runtime.selectgo(0xc000d06ed8, 0xc000d06e80, 0xc000d06ea8?, 0x0, 0xc000d06e90?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d06e48 sp=0xc000d06cf8 pc=0x48916b +github.com/rancher/opni/pkg/util.ServeHandler({0x5b426e0, 0xc001fee370}, {0x5b13c00?, 0xc000102ea0}, {0x5b2a6d0?, 0xc001f0e540}) + /home/ec2-user/opni/pkg/util/servers.go:21 +0x26a fp=0xc000d06f08 sp=0xc000d06e48 pc=0x176a94a +github.com/rancher/opni/pkg/gateway.(*GatewayHTTPServer).ListenAndServe.func2() + /home/ec2-user/opni/pkg/gateway/http.go:182 +0x1b9 fp=0xc000d06fa0 sp=0xc000d06f08 pc=0x22b8699 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc000d06fe0 sp=0xc000d06fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d06fe8 sp=0xc000d06fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 558 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 464 [select]: +runtime.gopark(0xc001533f70?, 0x2?, 0xb4?, 0x1b?, 0xc001533ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001533d20 sp=0xc001533d00 pc=0x4777ae +runtime.selectgo(0xc001533f70, 0xc001533eb8, 0x4aead9?, 0x0, 0x215774d?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001533e70 sp=0xc001533d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001493560, {0x5b426e0, 0xc0016635e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001533fa0 sp=0xc001533e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001533fe0 sp=0xc001533fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001533fe8 sp=0xc001533fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 462 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 465 [select]: +runtime.gopark(0xc001531f70?, 0x2?, 0xb4?, 0x1b?, 0xc001531ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001531d20 sp=0xc001531d00 pc=0x4777ae +runtime.selectgo(0xc001531f70, 0xc001531eb8, 0x4aead9?, 0x0, 0x5d5276?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001531e70 sp=0xc001531d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0014935c0, {0x5b426e0, 0xc001663630}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001531fa0 sp=0xc001531e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001531fe0 sp=0xc001531fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001531fe8 sp=0xc001531fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 462 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 601 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00148d9e0 sp=0xc00148d9c0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00148da18 sp=0xc00148d9e0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a490, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00148da38 sp=0xc00148da18 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0019e7fa0, 0x4b0501?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00148da80 sp=0xc00148da38 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0019e7f80) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc00148db70 sp=0xc00148da80 pc=0x513f65 +net.(*netFD).accept(0xc0019e7f80) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc00148dc68 sp=0xc00148db70 pc=0x8967be +net.(*TCPListener).accept(0xc001f0e540) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc00148dcb0 sp=0xc00148dc68 pc=0x8c383e +net.(*TCPListener).Accept(0xc001f0e540) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc00148dd20 sp=0xc00148dcb0 pc=0x8c1ba5 +net/http.(*onceCloseListener).Accept(0xc0017dd200) + :1 +0x44 fp=0xc00148dd58 sp=0xc00148dd20 pc=0xab5544 +net/http.(*Server).Serve(0xc001da40f0, {0x5b2a6d0, 0xc001f0e540}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc00148df50 sp=0xc00148dd58 pc=0xa7859f +github.com/rancher/opni/pkg/util.ServeHandler.func1() + /home/ec2-user/opni/pkg/util/servers.go:19 +0x47 fp=0xc00148dfa0 sp=0xc00148df50 pc=0x176aa67 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc00148dfe0 sp=0xc00148dfa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00148dfe8 sp=0xc00148dfe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 600 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 497 [select]: +runtime.gopark(0xc000c61f70?, 0x2?, 0xb4?, 0x1b?, 0xc000c61ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000c61d20 sp=0xc000c61d00 pc=0x4777ae +runtime.selectgo(0xc000c61f70, 0xc000c61eb8, 0xc000442070?, 0x0, 0x5b426e0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000c61e70 sp=0xc000c61d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001dec2c0, {0x5b426e0, 0xc0011524b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc000c61fa0 sp=0xc000c61e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc000c61fe0 sp=0xc000c61fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000c61fe8 sp=0xc000c61fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 496 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1225 [select]: +runtime.gopark(0xc001e33f10?, 0x3?, 0xb4?, 0x1b?, 0xc001e33e92?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e33ca8 sp=0xc001e33c88 pc=0x4777ae +runtime.selectgo(0xc001e33f10, 0xc001e33e8c, 0x5b126c0?, 0x0, 0x1?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e33df8 sp=0xc001e33ca8 pc=0x48916b +google.golang.org/grpc.(*addrConn).resetTransport(0xc000ebd500) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/clientconn.go:1368 +0x3ef fp=0xc001e33f50 sp=0xc001e33df8 pc=0x116334f +google.golang.org/grpc.(*addrConn).connect(0xc000ebd500) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/clientconn.go:993 +0x23b fp=0xc001e33fc0 sp=0xc001e33f50 pc=0x116033b +google.golang.org/grpc.(*acBalancerWrapper).Connect.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/balancer_conn_wrappers.go:398 +0x34 fp=0xc001e33fe0 sp=0xc001e33fc0 pc=0x1157434 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e33fe8 sp=0xc001e33fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*acBalancerWrapper).Connect in goroutine 542 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/balancer_conn_wrappers.go:398 +0xa7 + +goroutine 383 [select]: +runtime.gopark(0xc0018d2f80?, 0x3?, 0xb4?, 0x1b?, 0xc0018d2ee2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018d2d30 sp=0xc0018d2d10 pc=0x4777ae +runtime.selectgo(0xc0018d2f80, 0xc0018d2edc, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0018d2e80 sp=0xc0018d2d30 pc=0x48916b +github.com/lestrrat-go/backoff/v2.(*controller).loop(0xc001106180) + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:52 +0x178 fp=0xc0018d2fc0 sp=0xc0018d2e80 pc=0x13b0398 +github.com/lestrrat-go/backoff/v2.newController.func1() + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x34 fp=0xc0018d2fe0 sp=0xc0018d2fc0 pc=0x13b01f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018d2fe8 sp=0xc0018d2fe0 pc=0x4ad421 +created by github.com/lestrrat-go/backoff/v2.newController in goroutine 307 + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x4dd + +goroutine 514 [chan receive]: +runtime.gopark(0x4aef8c?, 0x4b0a55?, 0x28?, 0x60?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018d3e18 sp=0xc0018d3df8 pc=0x4777ae +runtime.chanrecv(0xc00119bb00, 0xc0018d3ed0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0018d3e90 sp=0xc0018d3e18 pc=0x441325 +runtime.chanrecv2(0xc000a46028?, 0xc00119ba40?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0018d3eb8 sp=0xc0018d3e90 pc=0x440f92 +github.com/nats-io/nats%2ego.(*Conn).flusher(0xc000a46000) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:3324 +0x1a5 fp=0xc0018d3fc0 sp=0xc0018d3eb8 pc=0x1f792e5 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x34 fp=0xc0018d3fe0 sp=0xc0018d3fc0 pc=0x1f714b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018d3fe8 sp=0xc0018d3fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 177 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x4cb + +goroutine 515 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc0015fff80?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019fcf08 sp=0xc0019fcee8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc000fe7bd0, 0x0) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc0019fcf58 sp=0xc0019fcf08 pc=0x4a9339 +sync.(*Cond).Wait(0xc000fe7bc0) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc0019fcf88 sp=0xc0019fcf58 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*asyncCallbacksHandler).asyncCBDispatcher(0xc0015fff80) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2842 +0x153 fp=0xc0019fcfc0 sp=0xc0019fcf88 pc=0x1f759b3 +github.com/nats-io/nats%2ego.Options.Connect.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x34 fp=0xc0019fcfe0 sp=0xc0019fcfc0 pc=0x1f6a534 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0019fcfe8 sp=0xc0019fcfe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.Options.Connect in goroutine 177 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x86b + +goroutine 516 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019fdf08 sp=0xc0019fdee8 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0019fdf80 sp=0xc0019fdf08 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0xc0019db7d0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0019fdfa8 sp=0xc0019fdf80 pc=0x440f72 +github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore.func3() + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:99 +0x4a fp=0xc0019fdfe0 sp=0xc0019fdfa8 pc=0x24a06aa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0019fdfe8 sp=0xc0019fdfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore in goroutine 177 + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:98 +0x4cb + +goroutine 517 [select]: +runtime.gopark(0xc00119bc90?, 0x13b0513?, 0x0?, 0x1?, 0xc0018d4ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018d4d30 sp=0xc0018d4d10 pc=0x4777ae +runtime.selectgo(0xc0018d4f60, 0xc0018d4ec8, 0x0?, 0x1, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0018d4e80 sp=0xc0018d4d30 pc=0x48916b +github.com/lestrrat-go/backoff/v2.(*controller).loop(0xc0011069c0) + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:64 +0x318 fp=0xc0018d4fc0 sp=0xc0018d4e80 pc=0x13b0538 +github.com/lestrrat-go/backoff/v2.newController.func1() + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x34 fp=0xc0018d4fe0 sp=0xc0018d4fc0 pc=0x13b01f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018d4fe8 sp=0xc0018d4fe0 pc=0x4ad421 +created by github.com/lestrrat-go/backoff/v2.newController in goroutine 177 + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x4dd + +goroutine 518 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x26?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018d0bd0 sp=0xc0018d0bb0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0018d0c08 sp=0xc0018d0bd0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162aa60, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0018d0c28 sp=0xc0018d0c08 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000a3dfa0, 0xc0019dc000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0018d0c70 sp=0xc0018d0c28 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000a3df80, {0xc0019dc000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0018d0d60 sp=0xc0018d0c70 pc=0x50e5a5 +net.(*netFD).Read(0xc000a3df80, {0xc0019dc000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0018d0dc0 sp=0xc0018d0d60 pc=0x89342b +net.(*conn).Read(0xc000b78778, {0xc0019dc000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0018d0e48 sp=0xc0018d0dc0 pc=0x8b176d +net.(*TCPConn).Read(0xc000b78778, {0xc0019dc000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc0018d0e90 sp=0xc0018d0e48 pc=0x8d72f4 +github.com/nats-io/nats%2ego.(*natsReader).Read(0xc000fe7e00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1890 +0x1a9 fp=0xc0018d0f10 sp=0xc0018d0e90 pc=0x1f6d169 +github.com/nats-io/nats%2ego.(*Conn).readLoop(0xc000a47c00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2917 +0x1f8 fp=0xc0018d0fc0 sp=0xc0018d0f10 pc=0x1f75fb8 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x34 fp=0xc0018d0fe0 sp=0xc0018d0fc0 pc=0x1f71514 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018d0fe8 sp=0xc0018d0fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 307 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x44e + +goroutine 519 [chan receive]: +runtime.gopark(0x4aef8c?, 0x4b0a55?, 0x28?, 0x7c?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019fbe18 sp=0xc0019fbdf8 pc=0x4777ae +runtime.chanrecv(0xc00119bda0, 0xc0019fbed0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0019fbe90 sp=0xc0019fbe18 pc=0x441325 +runtime.chanrecv2(0xc000a47c28?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0019fbeb8 sp=0xc0019fbe90 pc=0x440f92 +github.com/nats-io/nats%2ego.(*Conn).flusher(0xc000a47c00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:3324 +0x1a5 fp=0xc0019fbfc0 sp=0xc0019fbeb8 pc=0x1f792e5 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x34 fp=0xc0019fbfe0 sp=0xc0019fbfc0 pc=0x1f714b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0019fbfe8 sp=0xc0019fbfe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 307 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x4cb + +goroutine 520 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc000b741c0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019f7f08 sp=0xc0019f7ee8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc000fe7dd0, 0x0) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc0019f7f58 sp=0xc0019f7f08 pc=0x4a9339 +sync.(*Cond).Wait(0xc000fe7dc0) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc0019f7f88 sp=0xc0019f7f58 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*asyncCallbacksHandler).asyncCBDispatcher(0xc000b741c0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2842 +0x153 fp=0xc0019f7fc0 sp=0xc0019f7f88 pc=0x1f759b3 +github.com/nats-io/nats%2ego.Options.Connect.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x34 fp=0xc0019f7fe0 sp=0xc0019f7fc0 pc=0x1f6a534 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0019f7fe8 sp=0xc0019f7fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.Options.Connect in goroutine 307 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x86b + +goroutine 521 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc00123bc70?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018dddd8 sp=0xc0018dddb8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc00130c2d0, 0x15) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc0018dde28 sp=0xc0018dddd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc00130c2c0) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc0018dde58 sp=0xc0018dde28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc000a47c00, 0xc00123bc70) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc0018ddfb0 sp=0xc0018dde58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc0018ddfe0 sp=0xc0018ddfb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018ddfe8 sp=0xc0018ddfe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 307 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 522 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc0019fe0d0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018dfdd8 sp=0xc0018dfdb8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc00130c890, 0xa) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc0018dfe28 sp=0xc0018dfdd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc00130c880) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc0018dfe58 sp=0xc0018dfe28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc000a46000, 0xc0019fe0d0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc0018dffb0 sp=0xc0018dfe58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc0018dffe0 sp=0xc0018dffb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018dffe8 sp=0xc0018dffe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 177 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 502 [IO wait]: +runtime.gopark(0xc00152efe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00152ef98 sp=0xc00152ef78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00152efd0 sp=0xc00152ef98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6e530, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00152eff0 sp=0xc00152efd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0018bc920, 0xc000f15800?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00152f038 sp=0xc00152eff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0018bc900, {0xc000f15800, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00152f128 sp=0xc00152f038 pc=0x50e5a5 +net.(*netFD).Read(0xc0018bc900, {0xc000f15800, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00152f188 sp=0xc00152f128 pc=0x89342b +net.(*conn).Read(0xc000640cc0, {0xc000f15800, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00152f210 sp=0xc00152f188 pc=0x8b176d +net.(*UnixConn).Read(0xc000640cc0, {0xc000f15800, 0x600, 0x600}) + :1 +0x54 fp=0xc00152f258 sp=0xc00152f210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0001267c8, {0xc000f15800, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc00152f2b8 sp=0xc00152f258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000dc10a8, {0x5b0f6c0, 0xc0001267c8}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc00152f320 sp=0xc00152f2b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000dc0e00, {0x5b0f840?, 0xc000640cc0}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc00152f390 sp=0xc00152f320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000dc0e00, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc00152f9c0 sp=0xc00152f390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000dc0e00, {0xc001ad0000, 0x8000, 0xc000830550?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc00152faa8 sp=0xc00152f9c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0005fac00, {0xc001ad0000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc00152faf0 sp=0xc00152faa8 pc=0x10969ed +bufio.(*Reader).Read(0xc00128b0e0, {0xc001ae8040, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00152fbb0 sp=0xc00152faf0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00128b0e0}, {0xc001ae8040, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00152fbf8 sp=0xc00152fbb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc001ae8040, 0x9, 0x9}, {0x5b0f0a0, 0xc00128b0e0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00152fc98 sp=0xc00152fbf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001ae8000) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00152fdd0 sp=0xc00152fc98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000f94480, 0xc00128b200?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc00152ffb0 sp=0xc00152fdd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc00152ffe0 sp=0xc00152ffb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00152ffe8 sp=0xc00152ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 498 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 503 [select]: +runtime.gopark(0xc001af1e70?, 0x2?, 0x0?, 0x0?, 0xc001af1df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029d5c58 sp=0xc0029d5c38 pc=0x4777ae +runtime.selectgo(0xc0029d5e70, 0xc001af1df0, 0xc001af1e00?, 0x0, 0xc000dc0e00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029d5da8 sp=0xc0029d5c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000763e00, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0029d5ea0 sp=0xc0029d5da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001aab260) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0029d5f58 sp=0xc0029d5ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc0029d5fe0 sp=0xc0029d5f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029d5fe8 sp=0xc0029d5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 498 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 634 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000186830 sp=0xc000186810 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000186868 sp=0xc000186830 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a778, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000186888 sp=0xc000186868 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001651aa0, 0xc002066000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0001868d0 sp=0xc000186888 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001651a80, {0xc002066000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0001869c0 sp=0xc0001868d0 pc=0x50e5a5 +net.(*netFD).Read(0xc001651a80, {0xc002066000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc000186a20 sp=0xc0001869c0 pc=0x89342b +net.(*conn).Read(0xc001d6e0c0, {0xc002066000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc000186aa8 sp=0xc000186a20 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6e0c0, {0xc002066000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc000186af0 sp=0xc000186aa8 pc=0x8d72f4 +bufio.(*Reader).Read(0xc001f16ae0, {0xc0011d0660, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc000186bb0 sp=0xc000186af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001f16ae0}, {0xc0011d0660, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000186bf8 sp=0xc000186bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0011d0660, 0x9, 0x9}, {0x5b0f0a0, 0xc001f16ae0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc000186c98 sp=0xc000186bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011d0620) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc000186dd0 sp=0xc000186c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0014546c0, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc000186fb0 sp=0xc000186dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc000186fe0 sp=0xc000186fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000186fe8 sp=0xc000186fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 611 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 526 [select]: +runtime.gopark(0xc001bb7e48?, 0x2?, 0x0?, 0x0?, 0xc001bb7dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029d7c30 sp=0xc0029d7c10 pc=0x4777ae +runtime.selectgo(0xc0029d7e48, 0xc001bb7dc8, 0xc001bb7dd8?, 0x0, 0xc000dc1880?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029d7d80 sp=0xc0029d7c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0015fbc70, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0029d7e78 sp=0xc0029d7d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0008b6fc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0029d7f30 sp=0xc0029d7e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc0029d7fe0 sp=0xc0029d7f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029d7fe8 sp=0xc0029d7fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 501 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 528 [IO wait]: +runtime.gopark(0xc001ba6f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ba6f40 sp=0xc001ba6f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001ba6f78 sp=0xc001ba6f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6e340, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001ba6f98 sp=0xc001ba6f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0018bcc20, 0xc0014a1680?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001ba6fe0 sp=0xc001ba6f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0018bcc00, {0xc0014a1680, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001ba70d0 sp=0xc001ba6fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc0018bcc00, {0xc0014a1680, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001ba7130 sp=0xc001ba70d0 pc=0x89342b +net.(*conn).Read(0xc000640d18, {0xc0014a1680, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001ba71b8 sp=0xc001ba7130 pc=0x8b176d +net.(*UnixConn).Read(0xc000640d18, {0xc0014a1680, 0x240, 0x240}) + :1 +0x54 fp=0xc001ba7200 sp=0xc001ba71b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc000126798, {0xc0014a1680, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001ba7260 sp=0xc001ba7200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000dc1b28, {0x5b0f6c0, 0xc000126798}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc001ba72c8 sp=0xc001ba7260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000dc1880, {0x5b0f840?, 0xc000640d18}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001ba7338 sp=0xc001ba72c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000dc1880, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001ba7968 sp=0xc001ba7338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000dc1880, {0xc001b80000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001ba7a50 sp=0xc001ba7968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc000b74b60, {0xc001b80000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001ba7a98 sp=0xc001ba7a50 pc=0x10969ed +bufio.(*Reader).Read(0xc001107b60, {0xc0011d4740, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001ba7b58 sp=0xc001ba7a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001107b60}, {0xc0011d4740, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001ba7ba0 sp=0xc001ba7b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0011d4740, 0x9, 0x9}, {0x5b0f0a0, 0xc001107b60}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001ba7c40 sp=0xc001ba7ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011d4700) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001ba7d78 sp=0xc001ba7c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0019e4820, 0x0?, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001ba7ec8 sp=0xc001ba7d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc0010a4d20, {0x5b54e00?, 0xc0019e4820}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001ba7f80 sp=0xc001ba7ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001ba7fe0 sp=0xc001ba7f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ba7fe8 sp=0xc001ba7fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 501 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 545 [select]: +runtime.gopark(0xc001ddff70?, 0x2?, 0xb4?, 0x1b?, 0xc001ddfebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ddfd20 sp=0xc001ddfd00 pc=0x4777ae +runtime.selectgo(0xc001ddff70, 0xc001ddfeb8, 0xc00123f860?, 0x0, 0x440100?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ddfe70 sp=0xc001ddfd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000ee7860, {0x5b426e0, 0xc00010ff90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001ddffa0 sp=0xc001ddfe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001ddffe0 sp=0xc001ddffa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ddffe8 sp=0xc001ddffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 434 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 543 [select]: +runtime.gopark(0xc0018dbf70?, 0x2?, 0xb4?, 0x1b?, 0xc0018dbebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018dbd20 sp=0xc0018dbd00 pc=0x4777ae +runtime.selectgo(0xc0018dbf70, 0xc0018dbeb8, 0x0?, 0x0, 0x4aef8c?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0018dbe70 sp=0xc0018dbd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000ee7500, {0x5b426e0, 0xc00010f900}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0018dbfa0 sp=0xc0018dbe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0018dbfe0 sp=0xc0018dbfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018dbfe8 sp=0xc0018dbfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 434 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 562 [select]: +runtime.gopark(0xc001ba5f70?, 0x2?, 0xb4?, 0x1b?, 0xc001ba5ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ba5d20 sp=0xc001ba5d00 pc=0x4777ae +runtime.selectgo(0xc001ba5f70, 0xc001ba5eb8, 0xc0019e4ea0?, 0x0, 0x440940?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ba5e70 sp=0xc001ba5d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000ee78e0, {0x5b426e0, 0xc000b52000}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001ba5fa0 sp=0xc001ba5e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001ba5fe0 sp=0xc001ba5fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ba5fe8 sp=0xc001ba5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 434 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 548 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bc6f08 sp=0xc001bc6ee8 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001bc6f80 sp=0xc001bc6f08 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0x1134db0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001bc6fa8 sp=0xc001bc6f80 pc=0x440f72 +github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore.func3() + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:99 +0x4a fp=0xc001bc6fe0 sp=0xc001bc6fa8 pc=0x24a06aa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bc6fe8 sp=0xc001bc6fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore in goroutine 434 + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:98 +0x4cb + +goroutine 549 [select]: +runtime.gopark(0xc001b70070?, 0x13b0513?, 0x0?, 0x1?, 0xc001bc7ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bc7d30 sp=0xc001bc7d10 pc=0x4777ae +runtime.selectgo(0xc001bc7f60, 0xc001bc7ec8, 0x0?, 0x1, 0x448845?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001bc7e80 sp=0xc001bc7d30 pc=0x48916b +github.com/lestrrat-go/backoff/v2.(*controller).loop(0xc0012c8480) + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:64 +0x318 fp=0xc001bc7fc0 sp=0xc001bc7e80 pc=0x13b0538 +github.com/lestrrat-go/backoff/v2.newController.func1() + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x34 fp=0xc001bc7fe0 sp=0xc001bc7fc0 pc=0x13b01f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bc7fe8 sp=0xc001bc7fe0 pc=0x4ad421 +created by github.com/lestrrat-go/backoff/v2.newController in goroutine 434 + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x4dd + +goroutine 550 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc001170dd0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001579dd8 sp=0xc001579db8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc0012b5710, 0x9) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc001579e28 sp=0xc001579dd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc0012b5700) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc001579e58 sp=0xc001579e28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc001657c00, 0xc001170dd0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc001579fb0 sp=0xc001579e58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc001579fe0 sp=0xc001579fb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001579fe8 sp=0xc001579fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 434 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 551 [select]: +runtime.gopark(0xc001bc2f50?, 0x2?, 0xb4?, 0x1b?, 0xc001bc2eac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bc2ce8 sp=0xc001bc2cc8 pc=0x4777ae +runtime.selectgo(0xc001bc2f50, 0xc001bc2ea8, 0xc001bc2fa0?, 0x0, 0xc001170dd0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001bc2e38 sp=0xc001bc2ce8 pc=0x48916b +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).handleDriverNotifications(0xc0005a8c80) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:227 +0x125 fp=0xc001bc2fc0 sp=0xc001bc2e38 pc=0x3972405 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI.func2() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:89 +0x34 fp=0xc001bc2fe0 sp=0xc001bc2fc0 pc=0x396f0f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bc2fe8 sp=0xc001bc2fe0 pc=0x4ad421 +created by github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI in goroutine 177 + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:89 +0x54b + +goroutine 552 [select]: +runtime.gopark(0xc0020dbf40?, 0x3?, 0xb4?, 0x1b?, 0xc0020dbd8a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020dbb90 sp=0xc0020dbb70 pc=0x4777ae +runtime.selectgo(0xc0020dbf40, 0xc0020dbd84, 0x5?, 0x0, 0x40ed7c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020dbce0 sp=0xc0020dbb90 pc=0x48916b +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).runSync(0xc0005a8c80) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/admin.go:530 +0x80f fp=0xc0020dbfc0 sp=0xc0020dbce0 pc=0x396034f +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI.func3() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:90 +0x34 fp=0xc0020dbfe0 sp=0xc0020dbfc0 pc=0x396f094 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020dbfe8 sp=0xc0020dbfe0 pc=0x4ad421 +created by github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI in goroutine 177 + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:90 +0x5cb + +goroutine 1272 [select]: +runtime.gopark(0xc000a2ff10?, 0x3?, 0xb4?, 0x1b?, 0xc000a2fe92?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a2fca8 sp=0xc000a2fc88 pc=0x4777ae +runtime.selectgo(0xc000a2ff10, 0xc000a2fe8c, 0x5b126c0?, 0x0, 0x1?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a2fdf8 sp=0xc000a2fca8 pc=0x48916b +google.golang.org/grpc.(*addrConn).resetTransport(0xc000ebd800) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/clientconn.go:1368 +0x3ef fp=0xc000a2ff50 sp=0xc000a2fdf8 pc=0x116334f +google.golang.org/grpc.(*addrConn).connect(0xc000ebd800) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/clientconn.go:993 +0x23b fp=0xc000a2ffc0 sp=0xc000a2ff50 pc=0x116033b +google.golang.org/grpc.(*acBalancerWrapper).Connect.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/balancer_conn_wrappers.go:398 +0x34 fp=0xc000a2ffe0 sp=0xc000a2ffc0 pc=0x1157434 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a2ffe8 sp=0xc000a2ffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*acBalancerWrapper).Connect in goroutine 545 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/balancer_conn_wrappers.go:398 +0xa7 + +goroutine 564 [chan receive]: +runtime.gopark(0xc000b69f58?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bc5f00 sp=0xc001bc5ee0 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001bc5f78 sp=0xc001bc5f00 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0xc000b69f40?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001bc5fa0 sp=0xc001bc5f78 pc=0x440f72 +github.com/rancher/opni/plugins/metrics/pkg/cortex.NewClientSet.func1() + /home/ec2-user/opni/plugins/metrics/pkg/cortex/clientset.go:555 +0x53 fp=0xc001bc5fe0 sp=0xc001bc5fa0 pc=0x3d9cfb3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bc5fe8 sp=0xc001bc5fe0 pc=0x4ad421 +created by github.com/rancher/opni/plugins/metrics/pkg/cortex.NewClientSet in goroutine 434 + /home/ec2-user/opni/plugins/metrics/pkg/cortex/clientset.go:554 +0x999 + +goroutine 566 [select]: +runtime.gopark(0xc0011e5890?, 0x2?, 0x18?, 0x57?, 0xc0011e5844?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00157d6c0 sp=0xc00157d6a0 pc=0x4777ae +runtime.selectgo(0xc00157d890, 0xc0011e5840, 0xc001de8340?, 0x0, 0x100000001?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00157d810 sp=0xc00157d6c0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001dee000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc00157d8c0 sp=0xc00157d810 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001de8270, {0x42d9ca0?, 0xc0017e1200}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc00157db20 sp=0xc00157d8c0 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001d8bdf4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc00157db80 sp=0xc00157db20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001d8bd40, 0xc00157dd38, 0xc00157dd08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc00157dc28 sp=0xc00157db80 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8bd40, {0x42d9ca0, 0xc0017e1200}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc00157dd68 sp=0xc00157dc28 pc=0x119077e +github.com/rancher/opni/pkg/apis/management/v1.(*managementWatchClusterHealthStatusClient).Recv(0xc001dc4c10) + /home/ec2-user/opni/pkg/apis/management/v1/management_grpc.pb.go:252 +0x66 fp=0xc00157dda8 sp=0xc00157dd68 pc=0x177d086 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).watchGlobalClusterHealthStatus(0xc0005a8c80, {0x5b6f168?, 0xc000fe1300}, 0xc001b9d2c0) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/management.go:228 +0x642 fp=0xc00157df68 sp=0xc00157dda8 pc=0x3965de2 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).useWatchers.func2() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:247 +0x1be fp=0xc00157dfc0 sp=0xc00157df68 pc=0x3972fbe +github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents.func1() + /home/ec2-user/opni/pkg/management/watcher.go:27 +0x2f fp=0xc00157dfe0 sp=0xc00157dfc0 pc=0x23220cf +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00157dfe8 sp=0xc00157dfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents in goroutine 177 + /home/ec2-user/opni/pkg/management/watcher.go:26 +0x54 + +goroutine 567 [select]: +runtime.gopark(0xc0020e3f50?, 0x2?, 0xb4?, 0x1b?, 0xc0020e3e0c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020e3c48 sp=0xc0020e3c28 pc=0x4777ae +runtime.selectgo(0xc0020e3f50, 0xc0020e3e08, 0xc001dea380?, 0x0, 0xc0009d3ee0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020e3d98 sp=0xc0020e3c48 pc=0x48916b +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).watchCortexClusterStatus(0xc0005a8c80) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/management.go:70 +0x556 fp=0xc0020e3fa0 sp=0xc0020e3d98 pc=0x3963d76 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).useWatchers.func3() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:248 +0x2f fp=0xc0020e3fc0 sp=0xc0020e3fa0 pc=0x3972dcf +github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents.func1() + /home/ec2-user/opni/pkg/management/watcher.go:27 +0x2f fp=0xc0020e3fe0 sp=0xc0020e3fc0 pc=0x23220cf +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020e3fe8 sp=0xc0020e3fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents in goroutine 177 + /home/ec2-user/opni/pkg/management/watcher.go:26 +0x54 + +goroutine 568 [select]: +runtime.gopark(0xc000097fb0?, 0x2?, 0xb4?, 0x1b?, 0xc000097f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000097de8 sp=0xc000097dc8 pc=0x4777ae +runtime.selectgo(0xc000097fb0, 0xc000097f70, 0x1f75e37?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000097f38 sp=0xc000097de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc000097fe0 sp=0xc000097f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000097fe8 sp=0xc000097fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 565 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 569 [chan receive]: +runtime.gopark(0xc001519300?, 0xc001bd56e0?, 0x0?, 0x0?, 0xc0017dd080?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001519318 sp=0xc0015192f8 pc=0x4777ae +runtime.chanrecv(0xc001f16480, 0xc001519478, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001519390 sp=0xc001519318 pc=0x441325 +runtime.chanrecv2(0xc000bdc140?, 0x5b426a8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0015193b8 sp=0xc001519390 pc=0x440f92 +github.com/rancher/opni/pkg/management.(*Server).WatchClusters(0xc000d8a000, 0xc0017dd200, {0x5b4cb30, 0xc001bf1430}) + /home/ec2-user/opni/pkg/management/clusters.go:93 +0x26d fp=0xc0015194a8 sp=0xc0015193b8 pc=0x230a3cd +github.com/rancher/opni/pkg/apis/management/v1._Management_WatchClusters_Handler({0x4537040?, 0xc000d8a000}, {0x5b4ab30, 0xc0017dd1d0}) + /home/ec2-user/opni/pkg/apis/management/v1/management_grpc.pb.go:736 +0xfa fp=0xc001519508 sp=0xc0015194a8 pc=0x178093a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4537040, 0xc000d8a000}, {0x5b492f8, 0xc001b9d3b0}, 0xc000127c68, 0x47ec6c0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001519848 sp=0xc001519508 pc=0x12b0e02 +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef860, {0x5b54e00, 0xc001174680}, 0xc0019cbe60, 0xc000ff8960, 0x7716c60, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001519db0 sp=0xc001519848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef860, {0x5b54e00, 0xc001174680}, 0xc0019cbe60, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001519f28 sp=0xc001519db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001519fe0 sp=0xc001519f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001519fe8 sp=0xc001519fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 371 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 570 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc001f1a000?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0008b0dd8 sp=0xc0008b0db8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc0017e0890, 0x0) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc0008b0e28 sp=0xc0008b0dd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc0017e0880) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc0008b0e58 sp=0xc0008b0e28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc000c77100, 0xc001f1a000) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc0008b0fb0 sp=0xc0008b0e58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc0008b0fe0 sp=0xc0008b0fb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0008b0fe8 sp=0xc0008b0fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 569 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 571 [chan receive]: +runtime.gopark(0x3fa3c40?, 0xc001bd5860?, 0x0?, 0x0?, 0xc0019faf00?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019faf08 sp=0xc0019faee8 pc=0x4777ae +runtime.chanrecv(0xc001bd5860, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0019faf80 sp=0xc0019faf08 pc=0x441325 +runtime.chanrecv1(0xc001152190?, 0xc000c77100?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0019fafa8 sp=0xc0019faf80 pc=0x440f72 +github.com/nats-io/nats%2ego.(*js).subscribe.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/js.go:1817 +0x4a fp=0xc0019fafe0 sp=0xc0019fafa8 pc=0x1f4624a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0019fafe8 sp=0xc0019fafe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*js).subscribe in goroutine 569 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/js.go:1816 +0x2c85 + +goroutine 572 [select]: +runtime.gopark(0xc000a4ef90?, 0x2?, 0x50?, 0xed?, 0xc000a4ee94?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a4ece8 sp=0xc000a4ecc8 pc=0x4777ae +runtime.selectgo(0xc000a4ef90, 0xc000a4ee90, 0x5b11a20?, 0x0, 0xc00081e1c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a4ee38 sp=0xc000a4ece8 pc=0x48916b +github.com/rancher/opni/pkg/storage/jetstream.(*JetStreamStore).WatchClusters.func1() + /home/ec2-user/opni/pkg/storage/jetstream/cluster_store.go:247 +0x18c fp=0xc000a4efe0 sp=0xc000a4ee38 pc=0x249e4ac +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a4efe8 sp=0xc000a4efe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage/jetstream.(*JetStreamStore).WatchClusters in goroutine 569 + /home/ec2-user/opni/pkg/storage/jetstream/cluster_store.go:244 +0xbcd + +goroutine 670 [IO wait]: +runtime.gopark(0xc0024c7bf0?, 0x26?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024c7bd0 sp=0xc0024c7bb0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0024c7c08 sp=0xc0024c7bd0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241629dc8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0024c7c28 sp=0xc0024c7c08 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0014c8aa0, 0xc00188c000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0024c7c70 sp=0xc0024c7c28 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0014c8a80, {0xc00188c000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0024c7d60 sp=0xc0024c7c70 pc=0x50e5a5 +net.(*netFD).Read(0xc0014c8a80, {0xc00188c000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0024c7dc0 sp=0xc0024c7d60 pc=0x89342b +net.(*conn).Read(0xc001ce24d8, {0xc00188c000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0024c7e48 sp=0xc0024c7dc0 pc=0x8b176d +net.(*TCPConn).Read(0xc001ce24d8, {0xc00188c000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc0024c7e90 sp=0xc0024c7e48 pc=0x8d72f4 +github.com/nats-io/nats%2ego.(*natsReader).Read(0xc001641d00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1890 +0x1a9 fp=0xc0024c7f10 sp=0xc0024c7e90 pc=0x1f6d169 +github.com/nats-io/nats%2ego.(*Conn).readLoop(0xc0020ace00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2917 +0x1f8 fp=0xc0024c7fc0 sp=0xc0024c7f10 pc=0x1f75fb8 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x34 fp=0xc0024c7fe0 sp=0xc0024c7fc0 pc=0x1f71514 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024c7fe8 sp=0xc0024c7fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 609 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x44e + +goroutine 599 [select]: +runtime.gopark(0xc0008b1ee0?, 0x2?, 0xb4?, 0x1b?, 0xc0008b1e8c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0008b1d00 sp=0xc0008b1ce0 pc=0x4777ae +runtime.selectgo(0xc0008b1ee0, 0xc0008b1e88, 0x9?, 0x0, 0x5b426e0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0008b1e50 sp=0xc0008b1d00 pc=0x48916b +github.com/rancher/opni/pkg/util.ServeHandler({0x5b426e0, 0xc001fee370}, {0x5b13c00?, 0xc000102b60}, {0x5b2a5b0?, 0xc0010d99b0}) + /home/ec2-user/opni/pkg/util/servers.go:21 +0x26a fp=0xc0008b1f10 sp=0xc0008b1e50 pc=0x176a94a +github.com/rancher/opni/pkg/gateway.(*GatewayHTTPServer).ListenAndServe.func1() + /home/ec2-user/opni/pkg/gateway/http.go:178 +0x199 fp=0xc0008b1fa0 sp=0xc0008b1f10 pc=0x22b8879 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0008b1fe0 sp=0xc0008b1fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0008b1fe8 sp=0xc0008b1fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 558 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 593 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0016f3830 sp=0xc0016f3810 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0016f3868 sp=0xc0016f3830 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a0b0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0016f3888 sp=0xc0016f3868 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0014c8920, 0xc001f50000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0016f38d0 sp=0xc0016f3888 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0014c8900, {0xc001f50000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0016f39c0 sp=0xc0016f38d0 pc=0x50e5a5 +net.(*netFD).Read(0xc0014c8900, {0xc001f50000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0016f3a20 sp=0xc0016f39c0 pc=0x89342b +net.(*conn).Read(0xc000b79250, {0xc001f50000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0016f3aa8 sp=0xc0016f3a20 pc=0x8b176d +net.(*TCPConn).Read(0xc000b79250, {0xc001f50000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc0016f3af0 sp=0xc0016f3aa8 pc=0x8d72f4 +bufio.(*Reader).Read(0xc001655920, {0xc0011d4900, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0016f3bb0 sp=0xc0016f3af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001655920}, {0xc0011d4900, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0016f3bf8 sp=0xc0016f3bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0011d4900, 0x9, 0x9}, {0x5b0f0a0, 0xc001655920}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0016f3c98 sp=0xc0016f3bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011d48c0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0016f3dd0 sp=0xc0016f3c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001285200, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0016f3fb0 sp=0xc0016f3dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0016f3fe0 sp=0xc0016f3fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0016f3fe8 sp=0xc0016f3fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 587 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 613 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015e9988 sp=0xc0015e9968 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0015e99c0 sp=0xc0015e9988 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a680, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0015e99e0 sp=0xc0015e99c0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0019e7f20, 0xc000d8fb01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0015e9a28 sp=0xc0015e99e0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0019e7f00) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0015e9b18 sp=0xc0015e9a28 pc=0x513f65 +net.(*netFD).accept(0xc0019e7f00) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0015e9c10 sp=0xc0015e9b18 pc=0x8967be +net.(*TCPListener).accept(0xc001f0e4a0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc0015e9c58 sp=0xc0015e9c10 pc=0x8c383e +net.(*TCPListener).Accept(0xc001f0e4a0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc0015e9cc8 sp=0xc0015e9c58 pc=0x8c1ba5 +crypto/tls.(*listener).Accept(0xc0010d99b0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/tls.go:66 +0x43 fp=0xc0015e9d20 sp=0xc0015e9cc8 pc=0x9b2d43 +net/http.(*onceCloseListener).Accept(0xc0020b8ff0) + :1 +0x44 fp=0xc0015e9d58 sp=0xc0015e9d20 pc=0xab5544 +net/http.(*Server).Serve(0xc001b9c000, {0x5b2a5b0, 0xc0010d99b0}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc0015e9f50 sp=0xc0015e9d58 pc=0xa7859f +github.com/rancher/opni/pkg/util.ServeHandler.func1() + /home/ec2-user/opni/pkg/util/servers.go:19 +0x47 fp=0xc0015e9fa0 sp=0xc0015e9f50 pc=0x176aa67 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0015e9fe0 sp=0xc0015e9fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015e9fe8 sp=0xc0015e9fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 599 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 729 [select]: +runtime.gopark(0xc0016efda8?, 0x2?, 0x60?, 0xf8?, 0xc0016efa94?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0016ef818 sp=0xc0016ef7f8 pc=0x4777ae +runtime.selectgo(0xc0016efda8, 0xc0016efa90, 0x0?, 0x0, 0x1860a0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0016ef968 sp=0xc0016ef818 pc=0x48916b +github.com/hashicorp/go-plugin.Serve(0xc0010fe060) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:474 +0x2091 fp=0xc0016effc0 sp=0xc0016ef968 pc=0x21665b1 +github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins.func4() + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x34 fp=0xc0016effe0 sp=0xc0016effc0 pc=0x24cd234 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0016effe8 sp=0xc0016effe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins in goroutine 609 + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x76b + +goroutine 635 [select]: +runtime.gopark(0xc001f4fe70?, 0x2?, 0x0?, 0x0?, 0xc001f4fdf4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011e5c58 sp=0xc0011e5c38 pc=0x4777ae +runtime.selectgo(0xc0011e5e70, 0xc001f4fdf0, 0xc001f4fe00?, 0x0, 0xc001d6e0c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011e5da8 sp=0xc0011e5c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002088820, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0011e5ea0 sp=0xc0011e5da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0005e5570) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0011e5f58 sp=0xc0011e5ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc0011e5fe0 sp=0xc0011e5f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011e5fe8 sp=0xc0011e5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 611 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 615 [select]: +runtime.gopark(0xc001055e48?, 0x2?, 0x0?, 0x0?, 0xc001055dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00121dc30 sp=0xc00121dc10 pc=0x4777ae +runtime.selectgo(0xc00121de48, 0xc001055dc8, 0xc001055dd8?, 0x0, 0xc000990210?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00121dd80 sp=0xc00121dc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001152c80, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00121de78 sp=0xc00121dd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001bf21c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00121df30 sp=0xc00121de78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc00121dfe0 sp=0xc00121df30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00121dfe8 sp=0xc00121dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 614 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 616 [select]: +runtime.gopark(0xc0015eaf70?, 0x4?, 0x28?, 0xd8?, 0xc0015eadd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015eac00 sp=0xc0015eabe0 pc=0x4777ae +runtime.selectgo(0xc0015eaf70, 0xc0015eadd0, 0x40a16a0?, 0x0, 0xc00173ad50?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0015ead50 sp=0xc0015eac00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001aa9a00) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc0015eafc0 sp=0xc0015ead50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc0015eafe0 sp=0xc0015eafc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015eafe8 sp=0xc0015eafe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 614 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 617 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d077d8 sp=0xc000d077b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000d07810 sp=0xc000d077d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a398, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000d07830 sp=0xc000d07810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0016228a0, 0xc000f34000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000d07878 sp=0xc000d07830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001622880, {0xc000f34000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc000d07968 sp=0xc000d07878 pc=0x50e5a5 +net.(*netFD).Read(0xc001622880, {0xc000f34000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc000d079c8 sp=0xc000d07968 pc=0x89342b +net.(*conn).Read(0xc000990210, {0xc000f34000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc000d07a50 sp=0xc000d079c8 pc=0x8b176d +net.(*TCPConn).Read(0xc000990210, {0xc000f34000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc000d07a98 sp=0xc000d07a50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc001df9020, {0xc001effa80, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc000d07b58 sp=0xc000d07a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001df9020}, {0xc001effa80, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000d07ba0 sp=0xc000d07b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc001effa80, 0x9, 0x9}, {0x5b0f0a0, 0xc001df9020}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc000d07c40 sp=0xc000d07ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001effa40) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc000d07d78 sp=0xc000d07c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001aa9a00, 0x4208e0?, 0x7f429a430900?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc000d07ec8 sp=0xc000d07d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc000e0c3c0, {0x5b54e00?, 0xc001aa9a00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc000d07f80 sp=0xc000d07ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc000d07fe0 sp=0xc000d07f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d07fe8 sp=0xc000d07fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 614 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 606 [IO wait]: +runtime.gopark(0x0?, 0xc00172ab40?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020e5708 sp=0xc0020e56e8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0020e5740 sp=0xc0020e5708 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a2a0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0020e5760 sp=0xc0020e5740 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d72120, 0xc0011aa000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0020e57a8 sp=0xc0020e5760 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000d72100, {0xc0011aa000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0020e5898 sp=0xc0020e57a8 pc=0x50e5a5 +net.(*netFD).Read(0xc000d72100, {0xc0011aa000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0020e58f8 sp=0xc0020e5898 pc=0x89342b +net.(*conn).Read(0xc001ce2290, {0xc0011aa000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0020e5980 sp=0xc0020e58f8 pc=0x8b176d +net.(*TCPConn).Read(0xc001ce2290, {0xc0011aa000, 0x1000, 0x1000}) + :1 +0x54 fp=0xc0020e59c8 sp=0xc0020e5980 pc=0x8d72f4 +net/http.(*persistConn).Read(0xc001f186c0, {0xc0011aa000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1954 +0xf9 fp=0xc0020e5a48 sp=0xc0020e59c8 pc=0xa9c1d9 +bufio.(*Reader).fill(0xc001e9b9e0) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:113 +0x29a fp=0xc0020e5ad8 sp=0xc0020e5a48 pc=0x5f863a +bufio.(*Reader).Peek(0xc001e9b9e0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:151 +0xc7 fp=0xc0020e5b60 sp=0xc0020e5ad8 pc=0x5f88e7 +net/http.(*persistConn).readLoop(0xc001f186c0) + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:2118 +0x2db fp=0xc0020e5fc0 sp=0xc0020e5b60 pc=0xa9d41b +net/http.(*Transport).dialConn.func5() + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1776 +0x34 fp=0xc0020e5fe0 sp=0xc0020e5fc0 pc=0xa9aef4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020e5fe8 sp=0xc0020e5fe0 pc=0x4ad421 +created by net/http.(*Transport).dialConn in goroutine 586 + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1776 +0x24da + +goroutine 605 [IO wait]: +runtime.gopark(0xc001ba1650?, 0x4d1aca?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ba15e0 sp=0xc001ba15c0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001ba1618 sp=0xc001ba15e0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a1a8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001ba1638 sp=0xc001ba1618 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d73220, 0xc001e01000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001ba1680 sp=0xc001ba1638 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000d73200, {0xc001e01000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001ba1770 sp=0xc001ba1680 pc=0x50e5a5 +net.(*netFD).Read(0xc000d73200, {0xc001e01000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001ba17d0 sp=0xc001ba1770 pc=0x89342b +net.(*conn).Read(0xc001ce2288, {0xc001e01000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001ba1858 sp=0xc001ba17d0 pc=0x8b176d +net.(*TCPConn).Read(0xc001ce2288, {0xc001e01000, 0x1000, 0x1000}) + :1 +0x54 fp=0xc001ba18a0 sp=0xc001ba1858 pc=0x8d72f4 +net/http.(*connReader).Read(0xc0017dd920, {0xc001e01000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:791 +0x2b5 fp=0xc001ba1938 sp=0xc001ba18a0 pc=0xa68115 +bufio.(*Reader).fill(0xc001e9b980) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:113 +0x29a fp=0xc001ba19c8 sp=0xc001ba1938 pc=0x5f863a +bufio.(*Reader).Peek(0xc001e9b980, 0x4) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:151 +0xc7 fp=0xc001ba1a50 sp=0xc001ba19c8 pc=0x5f88e7 +net/http.(*conn).serve(0xc00172a2d0, {0x5b426a8, 0xc0017dd2f0}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:2044 +0xe7c fp=0xc001ba1fa0 sp=0xc001ba1a50 pc=0xa7221c +net/http.(*Server).Serve.func3() + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3086 +0x50 fp=0xc001ba1fe0 sp=0xc001ba1fa0 pc=0xa78af0 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ba1fe8 sp=0xc001ba1fe0 pc=0x4ad421 +created by net/http.(*Server).Serve in goroutine 601 + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3086 +0x80d + +goroutine 607 [select]: +runtime.gopark(0xc001555f90?, 0x2?, 0x10?, 0x5d?, 0xc001555e6c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029d1cc8 sp=0xc0029d1ca8 pc=0x4777ae +runtime.selectgo(0xc0029d1f90, 0xc001555e68, 0xc001fdac00?, 0x0, 0xc0017dd9e0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029d1e18 sp=0xc0029d1cc8 pc=0x48916b +net/http.(*persistConn).writeLoop(0xc001f186c0) + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:2421 +0x196 fp=0xc0029d1fc0 sp=0xc0029d1e18 pc=0xa9fe36 +net/http.(*Transport).dialConn.func6() + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1777 +0x34 fp=0xc0029d1fe0 sp=0xc0029d1fc0 pc=0xa9ae94 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029d1fe8 sp=0xc0029d1fe0 pc=0x4ad421 +created by net/http.(*Transport).dialConn in goroutine 586 + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1777 +0x256b + +goroutine 642 [select]: +runtime.gopark(0xc00111fe70?, 0x2?, 0x0?, 0x0?, 0xc00111fdf4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001555c58 sp=0xc001555c38 pc=0x4777ae +runtime.selectgo(0xc001555e70, 0xc00111fdf0, 0xc00111fe00?, 0x0, 0xc000b79250?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001555da8 sp=0xc001555c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00137c820, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001555ea0 sp=0xc001555da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0007f9b20) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001555f58 sp=0xc001555ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc001555fe0 sp=0xc001555f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001555fe8 sp=0xc001555fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 587 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 643 [select]: +runtime.gopark(0xc0011c5e48?, 0x2?, 0x0?, 0x0?, 0xc0011c5dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f2fc30 sp=0xc001f2fc10 pc=0x4777ae +runtime.selectgo(0xc001f2fe48, 0xc0011c5dc8, 0xc0005e56c0?, 0x0, 0xc000b79248?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001f2fd80 sp=0xc001f2fc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00137c910, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001f2fe78 sp=0xc001f2fd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0005e56c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001f2ff30 sp=0xc001f2fe78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc001f2ffe0 sp=0xc001f2ff30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f2ffe8 sp=0xc001f2ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 591 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 644 [select]: +runtime.gopark(0xc0018d6f70?, 0x4?, 0x10?, 0x0?, 0xc0018d6dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018d6c00 sp=0xc0018d6be0 pc=0x4777ae +runtime.selectgo(0xc0018d6f70, 0xc0018d6dd0, 0xc0018d6da8?, 0x0, 0xc0011d48c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0018d6d50 sp=0xc0018d6c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0012a16c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc0018d6fc0 sp=0xc0018d6d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc0018d6fe0 sp=0xc0018d6fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018d6fe8 sp=0xc0018d6fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 591 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 645 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000c5b7d8 sp=0xc000c5b7b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000c5b810 sp=0xc000c5b7d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241629fb8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000c5b830 sp=0xc000c5b810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0014c89a0, 0xc001f68000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000c5b878 sp=0xc000c5b830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0014c8980, {0xc001f68000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc000c5b968 sp=0xc000c5b878 pc=0x50e5a5 +net.(*netFD).Read(0xc0014c8980, {0xc001f68000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc000c5b9c8 sp=0xc000c5b968 pc=0x89342b +net.(*conn).Read(0xc000b79248, {0xc001f68000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc000c5ba50 sp=0xc000c5b9c8 pc=0x8b176d +net.(*TCPConn).Read(0xc000b79248, {0xc001f68000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc000c5ba98 sp=0xc000c5ba50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc001655a40, {0xc0011d49e0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc000c5bb58 sp=0xc000c5ba98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001655a40}, {0xc0011d49e0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000c5bba0 sp=0xc000c5bb58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0011d49e0, 0x9, 0x9}, {0x5b0f0a0, 0xc001655a40}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc000c5bc40 sp=0xc000c5bba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011d49a0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc000c5bd78 sp=0xc000c5bc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0012a16c0, 0x2c7d7d22676e6972?, 0x6f69746964646122?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc000c5bec8 sp=0xc000c5bd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc000e0c3c0, {0x5b54e00?, 0xc0012a16c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc000c5bf80 sp=0xc000c5bec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc000c5bfe0 sp=0xc000c5bf80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000c5bfe8 sp=0xc000c5bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 591 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 658 [IO wait]: +runtime.gopark(0x4?, 0xc0012abc30?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012abc48 sp=0xc0012abc28 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0012abc80 sp=0xc0012abc48 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241629ec0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0012abca0 sp=0xc0012abc80 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001646140, 0xc001800000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0012abce8 sp=0xc0012abca0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001646120, {0xc001800000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0012abdd8 sp=0xc0012abce8 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc001ce2378, {0xc001800000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc0012abe38 sp=0xc0012abdd8 pc=0x51ed0d +io.copyBuffer({0x5b0ede0, 0xc000013200}, {0x5b0eb40, 0xc001ce2378}, {0x0, 0x0, 0x0}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:430 +0x27e fp=0xc0012abee0 sp=0xc0012abe38 pc=0x50115e +io.Copy(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:389 +os/exec.(*Cmd).writerDescriptor.func1() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:560 +0x5b fp=0xc0012abf50 sp=0xc0012abee0 pc=0x6552db +os/exec.(*Cmd).Start.func2(0xc001f0ecc0) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:717 +0x3d fp=0xc0012abfb8 sp=0xc0012abf50 pc=0x65669d +os/exec.(*Cmd).Start.func3() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:729 +0x42 fp=0xc0012abfe0 sp=0xc0012abfb8 pc=0x656622 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012abfe8 sp=0xc0012abfe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 609 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:716 +0xf3a + +goroutine 659 [IO wait]: +runtime.gopark(0x4b05a9?, 0xc0012adc70?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012adc48 sp=0xc0012adc28 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0012adc80 sp=0xc0012adc48 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163b390, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0012adca0 sp=0xc0012adc80 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001646200, 0xc001814000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0012adce8 sp=0xc0012adca0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0016461e0, {0xc001814000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0012addd8 sp=0xc0012adce8 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc001ce2390, {0xc001814000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc0012ade38 sp=0xc0012addd8 pc=0x51ed0d +io.copyBuffer({0x5b0ede0, 0xc000013338}, {0x5b0eb40, 0xc001ce2390}, {0x0, 0x0, 0x0}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:430 +0x27e fp=0xc0012adee0 sp=0xc0012ade38 pc=0x50115e +io.Copy(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:389 +os/exec.(*Cmd).writerDescriptor.func1() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:560 +0x5b fp=0xc0012adf50 sp=0xc0012adee0 pc=0x6552db +os/exec.(*Cmd).Start.func2(0xc001f0ed00) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:717 +0x3d fp=0xc0012adfb8 sp=0xc0012adf50 pc=0x65669d +os/exec.(*Cmd).Start.func3() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:729 +0x42 fp=0xc0012adfe0 sp=0xc0012adfb8 pc=0x656622 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012adfe8 sp=0xc0012adfe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 609 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:716 +0xf3a + +goroutine 660 [select]: +runtime.gopark(0xc001bd4910?, 0x656a30?, 0x0?, 0x1?, 0xc00205be7c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00205bce0 sp=0xc00205bcc0 pc=0x4777ae +runtime.selectgo(0xc00205bf60, 0xc00205be78, 0x0?, 0x1, 0xc000f1d6e0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00205be30 sp=0xc00205bce0 pc=0x48916b +os/exec.(*Cmd).watchCtx(0xc00116e840, 0xc001bd4900) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:757 +0x10f fp=0xc00205bfb0 sp=0xc00205be30 pc=0x656a6f +os/exec.(*Cmd).Start.func4() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:743 +0x45 fp=0xc00205bfe0 sp=0xc00205bfb0 pc=0x6565a5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00205bfe8 sp=0xc00205bfe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 609 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:743 +0xf05 + +goroutine 661 [syscall]: +syscall.Syscall6(0x0?, 0x0?, 0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/syscall/syscall_linux.go:91 +0x3d fp=0xc001bc0d38 sp=0xc001bc0cb0 pc=0x4d1b1d +os.(*Process).blockUntilWaitable(0xc0009ea8a0) + /home/ec2-user/sdk/go1.21.0/src/os/wait_waitid.go:32 +0x9f fp=0xc001bc0e18 sp=0xc001bc0d38 pc=0x529c5f +os.(*Process).wait(0xc0009ea8a0) + /home/ec2-user/sdk/go1.21.0/src/os/exec_unix.go:22 +0x4b fp=0xc001bc0e98 sp=0xc001bc0e18 pc=0x51e0ab +os.(*Process).Wait(...) + /home/ec2-user/sdk/go1.21.0/src/os/exec.go:134 +os/exec.(*Cmd).Wait(0xc00116e840) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:890 +0xb9 fp=0xc001bc0f60 sp=0xc001bc0e98 pc=0x657279 +github.com/onsi/gomega/gexec.(*Session).monitorForExit(0xc0014cac00, 0x200a3636202d2020?) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:196 +0x3f fp=0xc001bc0fb0 sp=0xc001bc0f60 pc=0x233731f +github.com/onsi/gomega/gexec.Start.func1() + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:96 +0x45 fp=0xc001bc0fe0 sp=0xc001bc0fb0 pc=0x2336d65 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bc0fe8 sp=0xc001bc0fe0 pc=0x4ad421 +created by github.com/onsi/gomega/gexec.Start in goroutine 609 + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:96 +0x9eb + +goroutine 662 [chan receive]: +runtime.gopark(0x202020200a303231?, 0x20200a343131202d?, 0x20?, 0x20?, 0x3638202d20202020?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015e4e78 sp=0xc0015e4e58 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0015e4ef0 sp=0xc0015e4e78 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0x20200a343031202d?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0015e4f18 sp=0xc0015e4ef0 pc=0x440f72 +github.com/rancher/opni/pkg/test.(*Environment).startJetstream.func1() + /home/ec2-user/opni/pkg/test/environment.go:562 +0x69 fp=0xc0015e4f50 sp=0xc0015e4f18 pc=0x24b23e9 +github.com/rancher/opni/pkg/util/waitctx.restrictive.Go.func1() + /home/ec2-user/opni/pkg/util/waitctx/context.go:162 +0x93 fp=0xc0015e4fe0 sp=0xc0015e4f50 pc=0x178df73 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015e4fe8 sp=0xc0015e4fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/util/waitctx.restrictive.Go in goroutine 609 + /home/ec2-user/opni/pkg/util/waitctx/context.go:160 +0xeb + +goroutine 1121 [select]: +runtime.gopark(0xc00129ff40?, 0x3?, 0xb4?, 0x1b?, 0xc00129fd8a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00129fb90 sp=0xc00129fb70 pc=0x4777ae +runtime.selectgo(0xc00129ff40, 0xc00129fd84, 0x5?, 0x0, 0x40ed7c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00129fce0 sp=0xc00129fb90 pc=0x48916b +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).runSync(0xc00070a000) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/admin.go:530 +0x80f fp=0xc00129ffc0 sp=0xc00129fce0 pc=0x396034f +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI.func3() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:90 +0x34 fp=0xc00129ffe0 sp=0xc00129ffc0 pc=0x396f094 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00129ffe8 sp=0xc00129ffe0 pc=0x4ad421 +created by github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI in goroutine 886 + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:90 +0x5cb + +goroutine 1154 [select]: +runtime.gopark(0xc002ec1f40?, 0x3?, 0x0?, 0x1?, 0xc002ec1d2a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ec1b60 sp=0xc002ec1b40 pc=0x4777ae +runtime.selectgo(0xc002ec1f40, 0xc002ec1d24, 0xc002bfaf0b?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ec1cb0 sp=0xc002ec1b60 pc=0x48916b +github.com/rancher/opni/pkg/health.(*Monitor).WatchHealthStatus.func1() + /home/ec2-user/opni/pkg/health/monitor.go:166 +0x68e fp=0xc002ec1fe0 sp=0xc002ec1cb0 pc=0x207082e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ec1fe8 sp=0xc002ec1fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/health.(*Monitor).WatchHealthStatus in goroutine 1136 + /home/ec2-user/opni/pkg/health/monitor.go:150 +0x8da + +goroutine 1085 [select]: +runtime.gopark(0xc002172420?, 0x3?, 0xa0?, 0x9a?, 0xc000f20ada?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018d7808 sp=0xc0018d77e8 pc=0x4777ae +runtime.selectgo(0xc002172420, 0xc000f20ad4, 0x0?, 0x0, 0xc00128fef0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0018d7958 sp=0xc0018d7808 pc=0x48916b +reflect.rselect({0xc0018d7b40, 0x3, 0xc00294abd0?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:589 +0x345 fp=0xc0018d7a08 sp=0xc0018d7958 pc=0x4a8f05 +reflect.Select({0xc001702ee0, 0x3, 0x2?}) + /home/ec2-user/sdk/go1.21.0/src/reflect/value.go:3104 +0xb8a fp=0xc0018d7bd0 sp=0xc0018d7a08 pc=0x57c44a +github.com/rancher/opni/pkg/util.WaitAll({0x5b426e0, 0xc0007639a0}, 0xc00128fe70, {0xc0018d7db8, 0x2, 0x2}) + /home/ec2-user/opni/pkg/util/servers.go:51 +0x3ea fp=0xc0018d7d50 sp=0xc0018d7bd0 pc=0x176ae8a +github.com/rancher/opni/pkg/gateway.(*Gateway).ListenAndServe(0xc00135f200, {0x5b426e0, 0xc001fee820}) + /home/ec2-user/opni/pkg/gateway/gateway.go:347 +0x2ae fp=0xc0018d7dd8 sp=0xc0018d7d50 pc=0x22b25ae +github.com/rancher/opni/pkg/test.(*Environment).startGateway.func3(0x4b0571) + /home/ec2-user/opni/pkg/test/environment.go:1709 +0x18c fp=0xc0018d7f88 sp=0xc0018d7dd8 pc=0x24c538c +github.com/rancher/opni/pkg/plugins/hooks.onLoadingCompletedHook.Invoke(0xc001ded3c0, 0xc00158aec0?) + /home/ec2-user/opni/pkg/plugins/hooks/complete.go:10 +0x34 fp=0xc0018d7fa8 sp=0xc0018d7f88 pc=0x2182b94 +github.com/rancher/opni/pkg/plugins.(*PluginLoader).Complete.func1() + /home/ec2-user/opni/pkg/plugins/loader.go:329 +0x51 fp=0xc0018d7fe0 sp=0xc0018d7fa8 pc=0x2189711 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018d7fe8 sp=0xc0018d7fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins.(*PluginLoader).Complete in goroutine 609 + /home/ec2-user/opni/pkg/plugins/loader.go:329 +0x138 + +goroutine 668 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00205c830 sp=0xc00205c810 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00205c868 sp=0xc00205c830 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163b1a0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00205c888 sp=0xc00205c868 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00182c720, 0xc001830000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00205c8d0 sp=0xc00205c888 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00182c700, {0xc001830000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00205c9c0 sp=0xc00205c8d0 pc=0x50e5a5 +net.(*netFD).Read(0xc00182c700, {0xc001830000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00205ca20 sp=0xc00205c9c0 pc=0x89342b +net.(*conn).Read(0xc001ce24d0, {0xc001830000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00205caa8 sp=0xc00205ca20 pc=0x8b176d +net.(*TCPConn).Read(0xc001ce24d0, {0xc001830000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc00205caf0 sp=0xc00205caa8 pc=0x8d72f4 +bufio.(*Reader).Read(0xc000f1b0e0, {0xc000978660, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00205cbb0 sp=0xc00205caf0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000f1b0e0}, {0xc000978660, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00205cbf8 sp=0xc00205cbb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000978660, 0x9, 0x9}, {0x5b0f0a0, 0xc000f1b0e0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00205cc98 sp=0xc00205cbf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000978620) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00205cdd0 sp=0xc00205cc98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000f94b40, 0xc000f1b140?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc00205cfb0 sp=0xc00205cdd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc00205cfe0 sp=0xc00205cfb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00205cfe8 sp=0xc00205cfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 639 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 669 [select]: +runtime.gopark(0xc001887e70?, 0x2?, 0x0?, 0x0?, 0xc001887df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000ba9c58 sp=0xc000ba9c38 pc=0x4777ae +runtime.selectgo(0xc000ba9e70, 0xc001887df0, 0xc001887e00?, 0x0, 0xc001ce24d0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000ba9da8 sp=0xc000ba9c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001fef860, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc000ba9ea0 sp=0xc000ba9da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001bf39d0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc000ba9f58 sp=0xc000ba9ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc000ba9fe0 sp=0xc000ba9f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000ba9fe8 sp=0xc000ba9fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 639 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 676 [select]: +runtime.gopark(0xc0018d5f70?, 0x4?, 0x58?, 0x5c?, 0xc0018d5dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018d5c00 sp=0xc0018d5be0 pc=0x4777ae +runtime.selectgo(0xc0018d5f70, 0xc0018d5dd0, 0x40a16a0?, 0x0, 0xc00143a750?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0018d5d50 sp=0xc0018d5c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001d65ba0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc0018d5fc0 sp=0xc0018d5d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc0018d5fe0 sp=0xc0018d5fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018d5fe8 sp=0xc0018d5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 674 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 677 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0014897d8 sp=0xc0014897b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001489810 sp=0xc0014897d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163b0a8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001489830 sp=0xc001489810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001c9bca0, 0xc0017aa000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001489878 sp=0xc001489830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001c9bc80, {0xc0017aa000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001489968 sp=0xc001489878 pc=0x50e5a5 +net.(*netFD).Read(0xc001c9bc80, {0xc0017aa000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0014899c8 sp=0xc001489968 pc=0x89342b +net.(*conn).Read(0xc001d6f220, {0xc0017aa000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001489a50 sp=0xc0014899c8 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6f220, {0xc0017aa000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc001489a98 sp=0xc001489a50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc000bc2420, {0xc0008812a0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001489b58 sp=0xc001489a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000bc2420}, {0xc0008812a0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001489ba0 sp=0xc001489b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0008812a0, 0x9, 0x9}, {0x5b0f0a0, 0xc000bc2420}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001489c40 sp=0xc001489ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000881260) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001489d78 sp=0xc001489c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001d65ba0, 0x616d656863732422?, 0x3a70747468223a22?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001489ec8 sp=0xc001489d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc000e0c3c0, {0x5b54e00?, 0xc001d65ba0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001489f80 sp=0xc001489ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001489fe0 sp=0xc001489f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001489fe8 sp=0xc001489fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 674 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 671 [chan receive]: +runtime.gopark(0x4aef8c?, 0x4b0a55?, 0x28?, 0xce?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00205ae18 sp=0xc00205adf8 pc=0x4777ae +runtime.chanrecv(0xc001bd5b60, 0xc00205aed0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00205ae90 sp=0xc00205ae18 pc=0x441325 +runtime.chanrecv2(0xc0020ace28?, 0x202d202020200a39?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc00205aeb8 sp=0xc00205ae90 pc=0x440f92 +github.com/nats-io/nats%2ego.(*Conn).flusher(0xc0020ace00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:3324 +0x1a5 fp=0xc00205afc0 sp=0xc00205aeb8 pc=0x1f792e5 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x34 fp=0xc00205afe0 sp=0xc00205afc0 pc=0x1f714b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00205afe8 sp=0xc00205afe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 609 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x4cb + +goroutine 672 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc00200d6a0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f3cf08 sp=0xc000f3cee8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc001641cd0, 0x0) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc000f3cf58 sp=0xc000f3cf08 pc=0x4a9339 +sync.(*Cond).Wait(0xc001641cc0) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc000f3cf88 sp=0xc000f3cf58 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*asyncCallbacksHandler).asyncCBDispatcher(0xc00200d6a0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2842 +0x153 fp=0xc000f3cfc0 sp=0xc000f3cf88 pc=0x1f759b3 +github.com/nats-io/nats%2ego.Options.Connect.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x34 fp=0xc000f3cfe0 sp=0xc000f3cfc0 pc=0x1f6a534 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f3cfe8 sp=0xc000f3cfe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.Options.Connect in goroutine 609 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x86b + +goroutine 673 [chan receive]: +runtime.gopark(0x36202d202020200a?, 0x202d202020200a36?, 0x31?, 0x31?, 0x2020200a3137202d?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017a9f08 sp=0xc0017a9ee8 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0017a9f80 sp=0xc0017a9f08 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0x200a303231202d20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0017a9fa8 sp=0xc0017a9f80 pc=0x440f72 +github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore.func3() + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:99 +0x4a fp=0xc0017a9fe0 sp=0xc0017a9fa8 pc=0x24a06aa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017a9fe8 sp=0xc0017a9fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore in goroutine 609 + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:98 +0x4cb + +goroutine 690 [select]: +runtime.gopark(0xc001bd5e10?, 0x13b0513?, 0x0?, 0x1?, 0xc00205decc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00205dd30 sp=0xc00205dd10 pc=0x4777ae +runtime.selectgo(0xc00205df60, 0xc00205dec8, 0x202020200a353720?, 0x1, 0x20200a3638202d20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00205de80 sp=0xc00205dd30 pc=0x48916b +github.com/lestrrat-go/backoff/v2.(*controller).loop(0xc000f1b5c0) + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:64 +0x318 fp=0xc00205dfc0 sp=0xc00205de80 pc=0x13b0538 +github.com/lestrrat-go/backoff/v2.newController.func1() + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x34 fp=0xc00205dfe0 sp=0xc00205dfc0 pc=0x13b01f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00205dfe8 sp=0xc00205dfe0 pc=0x4ad421 +created by github.com/lestrrat-go/backoff/v2.newController in goroutine 609 + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x4dd + +goroutine 730 [select]: +runtime.gopark(0xc0017a6f48?, 0x2?, 0x20?, 0x6f?, 0xc0017a6f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017a6d80 sp=0xc0017a6d60 pc=0x4777ae +runtime.selectgo(0xc0017a6f48, 0xc0017a6f00, 0x0?, 0x0, 0x20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017a6ed0 sp=0xc0017a6d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).Recv(0xc0010e4930) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:124 +0xeb fp=0xc0017a6f78 sp=0xc0017a6ed0 pc=0x2154feb +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc000b538b0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc0017a6fc0 sp=0xc0017a6f78 pc=0x215759f +github.com/hashicorp/go-plugin.(*GRPCServer).Init.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x34 fp=0xc0017a6fe0 sp=0xc0017a6fc0 pc=0x215adf4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017a6fe8 sp=0xc0017a6fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*GRPCServer).Init in goroutine 729 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x89d + +goroutine 1140 [select]: +runtime.gopark(0xc000a2bf50?, 0x2?, 0xb4?, 0x1b?, 0xc000a2be0c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a2bc48 sp=0xc000a2bc28 pc=0x4777ae +runtime.selectgo(0xc000a2bf50, 0xc000a2be08, 0xc0018b8620?, 0x0, 0x400?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a2bd98 sp=0xc000a2bc48 pc=0x48916b +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).watchCortexClusterStatus(0xc00070a000) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/management.go:70 +0x556 fp=0xc000a2bfa0 sp=0xc000a2bd98 pc=0x3963d76 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).useWatchers.func3() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:248 +0x2f fp=0xc000a2bfc0 sp=0xc000a2bfa0 pc=0x3972dcf +github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents.func1() + /home/ec2-user/opni/pkg/management/watcher.go:27 +0x2f fp=0xc000a2bfe0 sp=0xc000a2bfc0 pc=0x23220cf +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a2bfe8 sp=0xc000a2bfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents in goroutine 886 + /home/ec2-user/opni/pkg/management/watcher.go:26 +0x54 + +goroutine 1139 [select]: +runtime.gopark(0xc002fa3480?, 0x2?, 0xc0?, 0x58?, 0xc002fa3404?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002fa3270 sp=0xc002fa3250 pc=0x4777ae +runtime.selectgo(0xc002fa3480, 0xc002fa3400, 0xc002fa3510?, 0x0, 0xc002170000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002fa33c0 sp=0xc002fa3270 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc002463a40, {0xc001e520d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc002fa34b0 sp=0xc002fa33c0 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002463a40, {0xc001e520d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc002fa3548 sp=0xc002fa34b0 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001e52060, {0xc001e520d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002fa3590 sp=0xc002fa3548 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001e52060}, {0xc001e520d0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002fa35d8 sp=0xc002fa3590 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0015f8c60, {0xc001e520d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002fa3648 sp=0xc002fa35d8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001e520c0, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002fa3718 sp=0xc002fa3648 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x0?, 0xc0015f8c60, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002fa3808 sp=0xc002fa3718 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001e520c0, {0x7f424347ae80, 0x779c980}, 0x687daa?, {0x0, 0x0}, {0x42d9ca0, 0xc00303cc00}, 0xc002f8a9c0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002fa38c0 sp=0xc002fa3808 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ee3380, {0x42d9ca0?, 0xc00303cc00}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc002fa3b20 sp=0xc002fa38c0 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0015f82f4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc002fa3b80 sp=0xc002fa3b20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc0015f8240, 0xc002fa3d38, 0xc002fa3d08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc002fa3c28 sp=0xc002fa3b80 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0015f8240, {0x42d9ca0, 0xc00303cc00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc002fa3d68 sp=0xc002fa3c28 pc=0x119077e +github.com/rancher/opni/pkg/apis/management/v1.(*managementWatchClusterHealthStatusClient).Recv(0xc000b6f4e0) + /home/ec2-user/opni/pkg/apis/management/v1/management_grpc.pb.go:252 +0x66 fp=0xc002fa3da8 sp=0xc002fa3d68 pc=0x177d086 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).watchGlobalClusterHealthStatus(0xc00070a000, {0x5b6f168?, 0xc001a7f950}, 0xc002b0dc20) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/management.go:228 +0x642 fp=0xc002fa3f68 sp=0xc002fa3da8 pc=0x3965de2 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).useWatchers.func2() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:247 +0x1be fp=0xc002fa3fc0 sp=0xc002fa3f68 pc=0x3972fbe +github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents.func1() + /home/ec2-user/opni/pkg/management/watcher.go:27 +0x2f fp=0xc002fa3fe0 sp=0xc002fa3fc0 pc=0x23220cf +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002fa3fe8 sp=0xc002fa3fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents in goroutine 886 + /home/ec2-user/opni/pkg/management/watcher.go:26 +0x54 + +goroutine 733 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0016f1940 sp=0xc0016f1920 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0016f1978 sp=0xc0016f1940 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163afb0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0016f1998 sp=0xc0016f1978 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00193a520, 0xc0016f1a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0016f19e0 sp=0xc0016f1998 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc00193a500) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0016f1ad0 sp=0xc0016f19e0 pc=0x513f65 +net.(*netFD).accept(0xc00193a500) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0016f1bc8 sp=0xc0016f1ad0 pc=0x8967be +net.(*UnixListener).accept(0xc0010e42d0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc0016f1c08 sp=0xc0016f1bc8 pc=0x8cedfe +net.(*UnixListener).Accept(0xc0010e42d0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc0016f1c78 sp=0xc0016f1c08 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc001ded720) + :1 +0x44 fp=0xc0016f1cb0 sp=0xc0016f1c78 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc00191c5a0, {0x5b2c6a0, 0xc001ded720}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc0016f1ee0 sp=0xc0016f1cb0 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCServer).Serve(0xc00135fb00, {0x5b2c6a0, 0xc001ded720}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:153 +0xdd fp=0xc0016f1f98 sp=0xc0016f1ee0 pc=0x215b33d +github.com/hashicorp/go-plugin.Serve.func6() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x62 fp=0xc0016f1fe0 sp=0xc0016f1f98 pc=0x2166b62 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0016f1fe8 sp=0xc0016f1fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.Serve in goroutine 729 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x1f67 + +goroutine 691 [chan receive]: +runtime.gopark(0x0?, 0x10?, 0xa0?, 0xf0?, 0xc001620dc4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019f6d10 sp=0xc0019f6cf0 pc=0x4777ae +runtime.chanrecv(0xc0010fe3c0, 0xc0019f6e40, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0019f6d88 sp=0xc0019f6d10 pc=0x441325 +runtime.chanrecv2(0x3b9aca00?, 0x40651e0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0019f6db0 sp=0xc0019f6d88 pc=0x440f92 +github.com/hashicorp/go-plugin.pidWait(0x1860a0?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/process.go:20 +0xdb fp=0xc0019f6e68 sp=0xc0019f6db0 pc=0x216033b +github.com/hashicorp/go-plugin.(*Client).reattach.func1(0xc00135f950?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:853 +0xd8 fp=0xc0019f6fb8 sp=0xc0019f6e68 pc=0x21528b8 +github.com/hashicorp/go-plugin.(*Client).reattach.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:862 +0x42 fp=0xc0019f6fe0 sp=0xc0019f6fb8 pc=0x21527a2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0019f6fe8 sp=0xc0019f6fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*Client).reattach in goroutine 707 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:846 +0x365 + +goroutine 1057 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc00237c600?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002400f08 sp=0xc002400ee8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc002722690, 0x0) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc002400f58 sp=0xc002400f08 pc=0x4a9339 +sync.(*Cond).Wait(0xc002722680) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc002400f88 sp=0xc002400f58 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*asyncCallbacksHandler).asyncCBDispatcher(0xc00237c600) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2842 +0x153 fp=0xc002400fc0 sp=0xc002400f88 pc=0x1f759b3 +github.com/nats-io/nats%2ego.Options.Connect.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x34 fp=0xc002400fe0 sp=0xc002400fc0 pc=0x1f6a534 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002400fe8 sp=0xc002400fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.Options.Connect in goroutine 823 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x86b + +goroutine 1442 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ba3d60 sp=0xc002ba3d40 pc=0x4777ae +runtime.chanrecv(0xc001f03080, 0xc002ba3e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002ba3dd8 sp=0xc002ba3d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002ba3e00 sp=0xc002ba3dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002ba3fe0 sp=0xc002ba3e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ba3fe8 sp=0xc002ba3fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 712 [select]: +runtime.gopark(0xc0020e7da8?, 0x2?, 0x60?, 0x78?, 0xc0020e7a94?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020e7818 sp=0xc0020e77f8 pc=0x4777ae +runtime.selectgo(0xc0020e7da8, 0xc0020e7a90, 0x0?, 0x0, 0x1860a0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020e7968 sp=0xc0020e7818 pc=0x48916b +github.com/hashicorp/go-plugin.Serve(0xc001305d40) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:474 +0x2091 fp=0xc0020e7fc0 sp=0xc0020e7968 pc=0x21665b1 +github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins.func4() + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x34 fp=0xc0020e7fe0 sp=0xc0020e7fc0 pc=0x24cd234 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020e7fe8 sp=0xc0020e7fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins in goroutine 609 + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x76b + +goroutine 692 [select]: +runtime.gopark(0xc00198df70?, 0x2?, 0xb4?, 0x1b?, 0xc00198debc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00198dd20 sp=0xc00198dd00 pc=0x4777ae +runtime.selectgo(0xc00198df70, 0xc00198deb8, 0xc001fef950?, 0x0, 0x202020200a353720?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00198de70 sp=0xc00198dd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001f0f480, {0x5b426e0, 0xc001fefb80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc00198dfa0 sp=0xc00198de70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc00198dfe0 sp=0xc00198dfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00198dfe8 sp=0xc00198dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 693 [select]: +runtime.gopark(0xc0016f5f70?, 0x2?, 0xb4?, 0x1b?, 0xc0016f5ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0016f5d20 sp=0xc0016f5d00 pc=0x4777ae +runtime.selectgo(0xc0016f5f70, 0xc0016f5eb8, 0x657473756c437265?, 0x0, 0x223a226570797422?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0016f5e70 sp=0xc0016f5d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001f0f4e0, {0x5b426e0, 0xc001fefbd0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0016f5fa0 sp=0xc0016f5e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0016f5fe0 sp=0xc0016f5fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0016f5fe8 sp=0xc0016f5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 713 [select]: +runtime.gopark(0xc001967f48?, 0x2?, 0x20?, 0x7f?, 0xc001967f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001967d80 sp=0xc001967d60 pc=0x4777ae +runtime.selectgo(0xc001967f48, 0xc001967f00, 0x0?, 0x0, 0x20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001967ed0 sp=0xc001967d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).Recv(0xc001267050) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:124 +0xeb fp=0xc001967f78 sp=0xc001967ed0 pc=0x2154feb +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc00106e460) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc001967fc0 sp=0xc001967f78 pc=0x215759f +github.com/hashicorp/go-plugin.(*GRPCServer).Init.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x34 fp=0xc001967fe0 sp=0xc001967fc0 pc=0x215adf4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001967fe8 sp=0xc001967fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*GRPCServer).Init in goroutine 712 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x89d + +goroutine 1138 [select]: +runtime.gopark(0xc000f77580?, 0x2?, 0xc0?, 0x72?, 0xc000f77504?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f77370 sp=0xc000f77350 pc=0x4777ae +runtime.selectgo(0xc000f77580, 0xc000f77500, 0x5b203e8?, 0x0, 0x2710?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000f774c0 sp=0xc000f77370 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc002462730, {0xc002c01ae0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc000f775b0 sp=0xc000f774c0 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002462730, {0xc002c01ae0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc000f77648 sp=0xc000f775b0 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002c01aa0, {0xc002c01ae0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc000f77690 sp=0xc000f77648 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002c01aa0}, {0xc002c01ae0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000f776d8 sp=0xc000f77690 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0003bfb00, {0xc002c01ae0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc000f77748 sp=0xc000f776d8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002c01ad0, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc000f77818 sp=0xc000f77748 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc0003bfb00, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc000f77908 sp=0xc000f77818 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002c01ad0, {0x7f424347ae80, 0x779c980}, 0xc000f77a30?, {0x0, 0x0}, {0x432c3a0, 0xc002570980}, 0x4aead9?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc000f779c0 sp=0xc000f77908 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ee31e0, {0x432c3a0?, 0xc002570980}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc000f77c20 sp=0xc000f779c0 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc002b2bdf4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc000f77c80 sp=0xc000f77c20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc002b2bd40, 0xc000f77e38, 0xc000f77e08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc000f77d28 sp=0xc000f77c80 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc002b2bd40, {0x432c3a0, 0xc002570980}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc000f77e68 sp=0xc000f77d28 pc=0x119077e +github.com/rancher/opni/pkg/apis/management/v1.(*managementWatchClustersClient).Recv(0xc000b6ee30) + /home/ec2-user/opni/pkg/apis/management/v1/management_grpc.pb.go:184 +0x66 fp=0xc000f77ea8 sp=0xc000f77e68 pc=0x177c726 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).watchGlobalCluster(0xc00070a000, {0x5b6f168, 0xc001a7f950}, 0xc001173950?) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/management.go:124 +0x1de fp=0xc000f77f70 sp=0xc000f77ea8 pc=0x396477e +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).useWatchers.func1() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:246 +0x55 fp=0xc000f77fc0 sp=0xc000f77f70 pc=0x3973035 +github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents.func1() + /home/ec2-user/opni/pkg/management/watcher.go:27 +0x2f fp=0xc000f77fe0 sp=0xc000f77fc0 pc=0x23220cf +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f77fe8 sp=0xc000f77fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents in goroutine 886 + /home/ec2-user/opni/pkg/management/watcher.go:26 +0x54 + +goroutine 1159 [select]: +runtime.gopark(0xc001ef4ee0?, 0x2?, 0xb4?, 0x1b?, 0xc001ef4e8c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ef4d00 sp=0xc001ef4ce0 pc=0x4777ae +runtime.selectgo(0xc001ef4ee0, 0xc001ef4e88, 0xc001ef4f90?, 0x0, 0xc00290d5f0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ef4e50 sp=0xc001ef4d00 pc=0x48916b +github.com/rancher/opni/pkg/util.ServeHandler({0x5b426e0, 0xc00139cf00}, {0x5b13c00?, 0xc0012c49c0}, {0x5b2a5b0?, 0xc0028fd4d0}) + /home/ec2-user/opni/pkg/util/servers.go:21 +0x26a fp=0xc001ef4f10 sp=0xc001ef4e50 pc=0x176a94a +github.com/rancher/opni/pkg/gateway.(*GatewayHTTPServer).ListenAndServe.func1() + /home/ec2-user/opni/pkg/gateway/http.go:178 +0x199 fp=0xc001ef4fa0 sp=0xc001ef4f10 pc=0x22b8879 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc001ef4fe0 sp=0xc001ef4fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ef4fe8 sp=0xc001ef4fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1087 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 742 [select]: +runtime.gopark(0xc001985fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001985f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001985de8 sp=0xc001985dc8 pc=0x4777ae +runtime.selectgo(0xc001985fb0, 0xc001985f70, 0xc001985fd0?, 0x0, 0xc0018b8150?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001985f38 sp=0xc001985de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001985fe0 sp=0xc001985f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001985fe8 sp=0xc001985fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 695 [select]: +runtime.gopark(0xc002056f48?, 0x2?, 0x20?, 0xa8?, 0xc002056f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002056d80 sp=0xc002056d60 pc=0x4777ae +runtime.selectgo(0xc002056f48, 0xc002056f00, 0xc002056f08?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002056ed0 sp=0xc002056d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).Recv(0xc00141d900) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:234 +0xeb fp=0xc002056f78 sp=0xc002056ed0 pc=0x2155d2b +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc001fefdb0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc002056fc0 sp=0xc002056f78 pc=0x215759f +github.com/hashicorp/go-plugin.newGRPCClient.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x34 fp=0xc002056fe0 sp=0xc002056fc0 pc=0x21599b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002056fe8 sp=0xc002056fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 707 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x5b7 + +goroutine 696 [select]: +runtime.gopark(0xc001751748?, 0x2?, 0xd0?, 0x15?, 0xc0017516fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001751578 sp=0xc001751558 pc=0x4777ae +runtime.selectgo(0xc001751748, 0xc0017516f8, 0xc001262460?, 0x0, 0xc00182cc18?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017516c8 sp=0xc001751578 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00165c000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001751778 sp=0xc0017516c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001589ee0, {0x43dd860?, 0xc000bdc960}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0017519d8 sp=0xc001751778 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001f19f14?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001751a38 sp=0xc0017519d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001f19e60, 0xc001751bf0, 0xc001751bc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc001751ae0 sp=0xc001751a38 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f19e60, {0x43dd860, 0xc000bdc960}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001751c20 sp=0xc001751ae0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc00134b7c0, {0x43dd860, 0xc000bdc960}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc001751de8 sp=0xc001751c20 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamClient).Recv(0xc00143a8d0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:132 +0x66 fp=0xc001751e28 sp=0xc001751de8 pc=0x2102606 +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream(0xc00141d900) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:197 +0x2e8 fp=0xc001751fc0 sp=0xc001751e28 pc=0x2155468 +github.com/hashicorp/go-plugin.newGRPCClient.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x34 fp=0xc001751fe0 sp=0xc001751fc0 pc=0x2159954 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001751fe8 sp=0xc001751fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 707 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x62e + +goroutine 716 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001995940 sp=0xc001995920 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001995978 sp=0xc001995940 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163aad8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001995998 sp=0xc001995978 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0014c9620, 0xc001995a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0019959e0 sp=0xc001995998 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0014c9600) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001995ad0 sp=0xc0019959e0 pc=0x513f65 +net.(*netFD).accept(0xc0014c9600) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001995bc8 sp=0xc001995ad0 pc=0x8967be +net.(*UnixListener).accept(0xc0012669f0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc001995c08 sp=0xc001995bc8 pc=0x8cedfe +net.(*UnixListener).Accept(0xc0012669f0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc001995c78 sp=0xc001995c08 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc000ee67a0) + :1 +0x44 fp=0xc001995cb0 sp=0xc001995c78 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc0004ef0e0, {0x5b2c6a0, 0xc000ee67a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc001995ee0 sp=0xc001995cb0 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCServer).Serve(0xc001173a70, {0x5b2c6a0, 0xc000ee67a0}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:153 +0xdd fp=0xc001995f98 sp=0xc001995ee0 pc=0x215b33d +github.com/hashicorp/go-plugin.Serve.func6() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x62 fp=0xc001995fe0 sp=0xc001995f98 pc=0x2166b62 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001995fe8 sp=0xc001995fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.Serve in goroutine 712 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x1f67 + +goroutine 1072 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029bd9e0 sp=0xc0029bd9c0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0029bda18 sp=0xc0029bd9e0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163a8e8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0029bda38 sp=0xc0029bda18 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001508320, 0x4b0501?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0029bda80 sp=0xc0029bda38 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc001508300) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0029bdb70 sp=0xc0029bda80 pc=0x513f65 +net.(*netFD).accept(0xc001508300) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0029bdc68 sp=0xc0029bdb70 pc=0x8967be +net.(*TCPListener).accept(0xc001331240) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc0029bdcb0 sp=0xc0029bdc68 pc=0x8c383e +net.(*TCPListener).Accept(0xc001331240) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc0029bdd20 sp=0xc0029bdcb0 pc=0x8c1ba5 +google.golang.org/grpc.(*Server).Serve(0xc00191c1e0, {0x5b2a6d0, 0xc001331240}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc0029bdf50 sp=0xc0029bdd20 pc=0x117789e +github.com/rancher/opni/pkg/management.(*Server).listenAndServeGrpc.func1() + /home/ec2-user/opni/pkg/management/server.go:213 +0x5e fp=0xc0029bdfa0 sp=0xc0029bdf50 pc=0x231e69e +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0029bdfe0 sp=0xc0029bdfa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029bdfe8 sp=0xc0029bdfe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1143 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 743 [select]: +runtime.gopark(0xc001964fb0?, 0x2?, 0xb0?, 0x4d?, 0xc001964ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001964d58 sp=0xc001964d38 pc=0x4777ae +runtime.selectgo(0xc001964fb0, 0xc001964ee0, 0xc001964f98?, 0x0, 0xc0010e58f0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001964ea8 sp=0xc001964d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001964fe0 sp=0xc001964ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001964fe8 sp=0xc001964fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 718 [chan receive]: +runtime.gopark(0x0?, 0x10?, 0xa0?, 0xf0?, 0xc000953664?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002058d10 sp=0xc002058cf0 pc=0x4777ae +runtime.chanrecv(0xc000bc3260, 0xc002058e40, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002058d88 sp=0xc002058d10 pc=0x441325 +runtime.chanrecv2(0x3b9aca00?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc002058db0 sp=0xc002058d88 pc=0x440f92 +github.com/hashicorp/go-plugin.pidWait(0x1860a0?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/process.go:20 +0xdb fp=0xc002058e68 sp=0xc002058db0 pc=0x216033b +github.com/hashicorp/go-plugin.(*Client).reattach.func1(0xc000b3d0b0?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:853 +0xd8 fp=0xc002058fb8 sp=0xc002058e68 pc=0x21528b8 +github.com/hashicorp/go-plugin.(*Client).reattach.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:862 +0x42 fp=0xc002058fe0 sp=0xc002058fb8 pc=0x21527a2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002058fe8 sp=0xc002058fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*Client).reattach in goroutine 717 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:846 +0x365 + +goroutine 740 [IO wait]: +runtime.gopark(0xc001988fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001988f98 sp=0xc001988f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001988fd0 sp=0xc001988f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163a600, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001988ff0 sp=0xc001988fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00193a6a0, 0xc0026bd800?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001989038 sp=0xc001988ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00193a680, {0xc0026bd800, 0x4800, 0x4800}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001989128 sp=0xc001989038 pc=0x50e5a5 +net.(*netFD).Read(0xc00193a680, {0xc0026bd800, 0x4800, 0x4800}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001989188 sp=0xc001989128 pc=0x89342b +net.(*conn).Read(0xc000991348, {0xc0026bd800, 0x4800, 0x4800}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001989210 sp=0xc001989188 pc=0x8b176d +net.(*UnixConn).Read(0xc000991348, {0xc0026bd800, 0x4800, 0x4800}) + :1 +0x54 fp=0xc001989258 sp=0xc001989210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc002e63f68, {0xc0026bd800, 0x4800, 0x4800}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0019892b8 sp=0xc001989258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000379ea8, {0x5b0f6c0, 0xc002e63f68}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc001989320 sp=0xc0019892b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000379c00, {0x5b0f840?, 0xc000991348}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001989390 sp=0xc001989320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000379c00, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0019899c0 sp=0xc001989390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000379c00, {0xc0018c2000, 0x8000, 0xc001989af8?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001989aa8 sp=0xc0019899c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc000ee7be0, {0xc0018c2000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001989af0 sp=0xc001989aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc00127b740, {0xc0011d58c0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001989bb0 sp=0xc001989af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00127b740}, {0xc0011d58c0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001989bf8 sp=0xc001989bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0011d58c0, 0x9, 0x9}, {0x5b0f0a0, 0xc00127b740}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001989c98 sp=0xc001989bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011d5880) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001989dd0 sp=0xc001989c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001285b00, 0xc00189f180?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc001989fb0 sp=0xc001989dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc001989fe0 sp=0xc001989fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001989fe8 sp=0xc001989fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 694 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 704 [select]: +runtime.gopark(0xc001993310?, 0x3?, 0x40?, 0x31?, 0xc001993292?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019930f0 sp=0xc0019930d0 pc=0x4777ae +runtime.selectgo(0xc001993310, 0xc00199328c, 0x46216a2?, 0x0, 0x4?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001993240 sp=0xc0019930f0 pc=0x48916b +github.com/hashicorp/go-plugin.(*grpcStdioServer).StreamStdio(0xc00158b400, 0x42219c0?, {0x5b4ebb0, 0xc00114cec0}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:61 +0x1e5 fp=0xc001993350 sp=0xc001993240 pc=0x215bbc5 +github.com/hashicorp/go-plugin/internal/plugin._GRPCStdio_StreamStdio_Handler({0x409e580?, 0xc00158b400}, {0x5b4a6f0, 0xc001f0f9e0}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:185 +0xf7 fp=0xc0019933b0 sp=0xc001993350 pc=0x2104077 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x409e580, 0xc00158b400}, {0x5b4ab30, 0xc001c0a3f0}, 0xc0001a1818?, 0x47efca8) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001993400 sp=0xc0019933b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x409e580, 0xc00158b400}, {0x5b4ab30, 0xc001c0a3f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001993490 sp=0xc001993400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x409e580, 0xc00158b400}, {0x5b492f8, 0xc001da41e0}, 0xc0001a1818, 0xc00141dec0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0019937d0 sp=0xc001993490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x409e580, 0xc00158b400}, {0x5b492f8, 0xc001da41e0}, 0x3f252a0?, 0xc00114cdf0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001993848 sp=0xc0019937d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000853200, 0xc0010e4ba0, 0x770cba0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001993db0 sp=0xc001993848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000853200, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001993f28 sp=0xc001993db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001993fe0 sp=0xc001993f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001993fe8 sp=0xc001993fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 719 [select]: +runtime.gopark(0xc001f45f70?, 0x2?, 0xb4?, 0x1b?, 0xc001f45ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f45d20 sp=0xc001f45d00 pc=0x4777ae +runtime.selectgo(0xc001f45f70, 0xc001f45eb8, 0x736c61663a227365?, 0x0, 0x7463656a626f223a?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001f45e70 sp=0xc001f45d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000ee7580, {0x5b426e0, 0xc00106e910}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001f45fa0 sp=0xc001f45e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001f45fe0 sp=0xc001f45fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f45fe8 sp=0xc001f45fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 720 [select]: +runtime.gopark(0xc001de3f70?, 0x2?, 0xb4?, 0x1b?, 0xc001de3ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001de3d20 sp=0xc001de3d00 pc=0x4777ae +runtime.selectgo(0xc001de3f70, 0xc001de3eb8, 0x63732f34302d7466?, 0x0, 0x23223a2266657224?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001de3e70 sp=0xc001de3d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000ee7600, {0x5b426e0, 0xc00106e960}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001de3fa0 sp=0xc001de3e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001de3fe0 sp=0xc001de3fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001de3fe8 sp=0xc001de3fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 699 [select]: +runtime.gopark(0xc0017a8f48?, 0x2?, 0x50?, 0x8d?, 0xc0017a8f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017a8d80 sp=0xc0017a8d60 pc=0x4777ae +runtime.selectgo(0xc0017a8f48, 0xc0017a8f00, 0x8000?, 0x0, 0xc001266cf0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017a8ed0 sp=0xc0017a8d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).Recv(0xc00141da40) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:234 +0xeb fp=0xc0017a8f78 sp=0xc0017a8ed0 pc=0x2155d2b +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc0011f24b0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc0017a8fc0 sp=0xc0017a8f78 pc=0x215759f +github.com/hashicorp/go-plugin.newGRPCClient.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x34 fp=0xc0017a8fe0 sp=0xc0017a8fc0 pc=0x21599b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017a8fe8 sp=0xc0017a8fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 717 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x5b7 + +goroutine 687 [select]: +runtime.gopark(0xc0017d2fb0?, 0x2?, 0xb4?, 0x1b?, 0xc0017d2f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017d2de8 sp=0xc0017d2dc8 pc=0x4777ae +runtime.selectgo(0xc0017d2fb0, 0xc0017d2f70, 0x1179540?, 0x0, 0xc0017d04e0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017d2f38 sp=0xc0017d2de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0017d2fe0 sp=0xc0017d2f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017d2fe8 sp=0xc0017d2fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 700 [select]: +runtime.gopark(0xc00197b748?, 0x2?, 0xd0?, 0xb5?, 0xc00197b6fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00197b578 sp=0xc00197b558 pc=0x4777ae +runtime.selectgo(0xc00197b748, 0xc00197b6f8, 0xc0012626a0?, 0x0, 0xc00182d118?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00197b6c8 sp=0xc00197b578 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0015f99e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc00197b778 sp=0xc00197b6c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00184a340, {0x43dd860?, 0xc00128c730}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc00197b9d8 sp=0xc00197b778 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000852534?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc00197ba38 sp=0xc00197b9d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc000852480, 0xc00197bbf0, 0xc00197bbc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc00197bae0 sp=0xc00197ba38 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc000852480, {0x43dd860, 0xc00128c730}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc00197bc20 sp=0xc00197bae0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc0011615c0, {0x43dd860, 0xc00128c730}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc00197bde8 sp=0xc00197bc20 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamClient).Recv(0xc00158ba70) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:132 +0x66 fp=0xc00197be28 sp=0xc00197bde8 pc=0x2102606 +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream(0xc00141da40) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:197 +0x2e8 fp=0xc00197bfc0 sp=0xc00197be28 pc=0x2155468 +github.com/hashicorp/go-plugin.newGRPCClient.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x34 fp=0xc00197bfe0 sp=0xc00197bfc0 pc=0x2159954 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00197bfe8 sp=0xc00197bfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 717 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x62e + +goroutine 688 [select]: +runtime.gopark(0xc0017a2fb0?, 0x2?, 0xb0?, 0x2d?, 0xc0017a2ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017a2d58 sp=0xc0017a2d38 pc=0x4777ae +runtime.selectgo(0xc0017a2fb0, 0xc0017a2ee0, 0xc0017a2f98?, 0x0, 0xc0012b6030?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017a2ea8 sp=0xc0017a2d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0017a2fe0 sp=0xc0017a2ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017a2fe8 sp=0xc0017a2fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 736 [IO wait]: +runtime.gopark(0xc001f46fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f46f98 sp=0xc001f46f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001f46fd0 sp=0xc001f46f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163c3f0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001f46ff0 sp=0xc001f46fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0014c9a20, 0xc00298a000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001f47038 sp=0xc001f46ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0014c9a00, {0xc00298a000, 0xa000, 0xa000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001f47128 sp=0xc001f47038 pc=0x50e5a5 +net.(*netFD).Read(0xc0014c9a00, {0xc00298a000, 0xa000, 0xa000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001f47188 sp=0xc001f47128 pc=0x89342b +net.(*conn).Read(0xc000b79a60, {0xc00298a000, 0xa000, 0xa000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001f47210 sp=0xc001f47188 pc=0x8b176d +net.(*UnixConn).Read(0xc000b79a60, {0xc00298a000, 0xa000, 0xa000}) + :1 +0x54 fp=0xc001f47258 sp=0xc001f47210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0032a2a50, {0xc00298a000, 0xa000, 0xa000}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001f472b8 sp=0xc001f47258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc00189f428, {0x5b0f6c0, 0xc0032a2a50}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc001f47320 sp=0xc001f472b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc00189f180, {0x5b0f840?, 0xc000b79a60}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001f47390 sp=0xc001f47320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc00189f180, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001f479c0 sp=0xc001f47390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc00189f180, {0xc0019e8000, 0x8000, 0xc003300370?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001f47aa8 sp=0xc001f479c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001dedc00, {0xc0019e8000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001f47af0 sp=0xc001f47aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc001db0f60, {0xc001938820, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001f47bb0 sp=0xc001f47af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001db0f60}, {0xc001938820, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001f47bf8 sp=0xc001f47bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc001938820, 0x9, 0x9}, {0x5b0f0a0, 0xc001db0f60}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001f47c98 sp=0xc001f47bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0019387e0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001f47dd0 sp=0xc001f47c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001c6c480, 0xc0004ef0e0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc001f47fb0 sp=0xc001f47dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc001f47fe0 sp=0xc001f47fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f47fe8 sp=0xc001f47fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 721 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 741 [select]: +runtime.gopark(0xc001af1e70?, 0x2?, 0x0?, 0x0?, 0xc001af1df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001af1c58 sp=0xc001af1c38 pc=0x4777ae +runtime.selectgo(0xc001af1e70, 0xc001af1df0, 0xc001af1e00?, 0x0, 0xc000379c00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001af1da8 sp=0xc001af1c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0007c00f0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001af1ea0 sp=0xc001af1da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0018b8150) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001af1f58 sp=0xc001af1ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc001af1fe0 sp=0xc001af1f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001af1fe8 sp=0xc001af1fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 694 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 701 [select]: +runtime.gopark(0xc00264de48?, 0x2?, 0x0?, 0x0?, 0xc00264ddcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00264dc30 sp=0xc00264dc10 pc=0x4777ae +runtime.selectgo(0xc00264de48, 0xc00264ddc8, 0xc00264ddd8?, 0x0, 0xc001520380?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00264dd80 sp=0xc00264dc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0011f3220, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00264de78 sp=0xc00264dd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0020127e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00264df30 sp=0xc00264de78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc00264dfe0 sp=0xc00264df30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00264dfe8 sp=0xc00264dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 697 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 702 [select]: +runtime.gopark(0xc001966f70?, 0x4?, 0xc0?, 0x2?, 0xc001966dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001966c00 sp=0xc001966be0 pc=0x4777ae +runtime.selectgo(0xc001966f70, 0xc001966dd0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001966d50 sp=0xc001966c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00183b520) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc001966fc0 sp=0xc001966d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc001966fe0 sp=0xc001966fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001966fe8 sp=0xc001966fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 697 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 703 [IO wait]: +runtime.gopark(0xc001990f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001990f40 sp=0xc001990f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001990f78 sp=0xc001990f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163a508, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001990f98 sp=0xc001990f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00182cd20, 0xc0011a9000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001990fe0 sp=0xc001990f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00182cd00, {0xc0011a9000, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0019910d0 sp=0xc001990fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc00182cd00, {0xc0011a9000, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001991130 sp=0xc0019910d0 pc=0x89342b +net.(*conn).Read(0xc001ce2600, {0xc0011a9000, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0019911b8 sp=0xc001991130 pc=0x8b176d +net.(*UnixConn).Read(0xc001ce2600, {0xc0011a9000, 0x800, 0x800}) + :1 +0x54 fp=0xc001991200 sp=0xc0019911b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc002e63f50, {0xc0011a9000, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001991260 sp=0xc001991200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc001520628, {0x5b0f6c0, 0xc002e63f50}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0019912c8 sp=0xc001991260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc001520380, {0x5b0f840?, 0xc001ce2600}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001991338 sp=0xc0019912c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc001520380, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001991968 sp=0xc001991338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc001520380, {0xc0019a8000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001991a50 sp=0xc001991968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001f0f900, {0xc0019a8000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001991a98 sp=0xc001991a50 pc=0x10969ed +bufio.(*Reader).Read(0xc0014828a0, {0xc000978740, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001991b58 sp=0xc001991a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0014828a0}, {0xc000978740, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001991ba0 sp=0xc001991b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000978740, 0x9, 0x9}, {0x5b0f0a0, 0xc0014828a0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001991c40 sp=0xc001991ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000978700) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001991d78 sp=0xc001991c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00183b520, 0x78?, 0x1766418d00000000?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001991ec8 sp=0xc001991d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c5a0, {0x5b54e00?, 0xc00183b520}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001991f80 sp=0xc001991ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001991fe0 sp=0xc001991f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001991fe8 sp=0xc001991fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 697 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 744 [chan receive]: +runtime.gopark(0xc001981d80?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001981d60 sp=0xc001981d40 pc=0x4777ae +runtime.chanrecv(0xc001713800, 0xc001981e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001981dd8 sp=0xc001981d60 pc=0x441325 +runtime.chanrecv1(0x1?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001981e00 sp=0xc001981dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001981fe0 sp=0xc001981e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001981fe8 sp=0xc001981fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 680 [select]: +runtime.gopark(0xc001960fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001960f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001960de8 sp=0xc001960dc8 pc=0x4777ae +runtime.selectgo(0xc001960fb0, 0xc001960f70, 0x1deeab63?, 0x0, 0xc0012669f0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001960f38 sp=0xc001960de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001960fe0 sp=0xc001960f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001960fe8 sp=0xc001960fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 696 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 681 [select]: +runtime.gopark(0xc0017d9fb0?, 0x2?, 0xb0?, 0x9d?, 0xc0017d9ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017d9d58 sp=0xc0017d9d38 pc=0x4777ae +runtime.selectgo(0xc0017d9fb0, 0xc0017d9ee0, 0xa3736202d202020?, 0x0, 0x448845?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017d9ea8 sp=0xc0017d9d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0017d9fe0 sp=0xc0017d9ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017d9fe8 sp=0xc0017d9fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 696 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 682 [chan receive]: +runtime.gopark(0x3332343637313138?, 0x7d2265686361632f?, 0x2c?, 0x22?, 0x223a22656e69676e?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001961d60 sp=0xc001961d40 pc=0x4777ae +runtime.chanrecv(0xc000f27f80, 0xc001961e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001961dd8 sp=0xc001961d60 pc=0x441325 +runtime.chanrecv1(0x706d742f223a2268?, 0x65742d696e706f2f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001961e00 sp=0xc001961dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001961fe0 sp=0xc001961e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001961fe8 sp=0xc001961fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 696 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 683 [select]: +runtime.gopark(0xc00198bfa0?, 0x3?, 0xf0?, 0xbd?, 0xc00198bf3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00198bda8 sp=0xc00198bd88 pc=0x4777ae +runtime.selectgo(0xc00198bfa0, 0xc00198bf34, 0xc001962f40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00198bef8 sp=0xc00198bda8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:184 +0x1b3 fp=0xc00198bfe0 sp=0xc00198bef8 pc=0x21557f3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00198bfe8 sp=0xc00198bfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream in goroutine 696 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:182 +0x24a + +goroutine 745 [select]: +runtime.gopark(0xc001a09790?, 0x2?, 0x18?, 0x96?, 0xc001a09744?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a095c0 sp=0xc001a095a0 pc=0x4777ae +runtime.selectgo(0xc001a09790, 0xc001a09740, 0xa10127972746e45?, 0x0, 0x1279656b03520928?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001a09710 sp=0xc001a095c0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00050e000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001a097c0 sp=0xc001a09710 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001589e10, {0x43c4ea0?, 0xc0012b4500}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc001a09a20 sp=0xc001a097c0 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001f19df4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001a09a80 sp=0xc001a09a20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001f19d40, 0xc0017d6c38, 0xc0017d6c08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc001a09b28 sp=0xc001a09a80 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f19d40, {0x43c4ea0, 0xc0012b4500}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001a09c68 sp=0xc001a09b28 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc0012b4280, {0x43c4ea0, 0xc0012b4500}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc001a09e30 sp=0xc001a09c68 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCStdioStreamStdioClient).Recv(0xc001511540) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:160 +0x66 fp=0xc001a09e70 sp=0xc001a09e30 pc=0x2103f26 +github.com/hashicorp/go-plugin.(*grpcStdioClient).Run(0xc000ee7de0, {0x5b0ee60, 0x779c980}, {0x5b0eb80, 0xc00007c038}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:135 +0x11c fp=0xc001a09f80 sp=0xc001a09e70 pc=0x215c15c +github.com/hashicorp/go-plugin.newGRPCClient.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x6c fp=0xc001a09fe0 sp=0xc001a09f80 pc=0x21598ec +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a09fe8 sp=0xc001a09fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 707 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x84f + +goroutine 737 [select]: +runtime.gopark(0xc00256be70?, 0x2?, 0x0?, 0x0?, 0xc00256bdf4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00256bc58 sp=0xc00256bc38 pc=0x4777ae +runtime.selectgo(0xc00256be70, 0xc00256bdf0, 0xc00202ddc0?, 0x0, 0xc00189f180?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00256bda8 sp=0xc00256bc58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00128c5f0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00256bea0 sp=0xc00256bda8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00202ddc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00256bf58 sp=0xc00256bea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc00256bfe0 sp=0xc00256bf58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00256bfe8 sp=0xc00256bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 721 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 746 [select]: +runtime.gopark(0xc001986fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001986f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001986de8 sp=0xc001986dc8 pc=0x4777ae +runtime.selectgo(0xc001986fb0, 0xc001986f70, 0x6b4344646b514e52?, 0x0, 0x6d65356355597a5a?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001986f38 sp=0xc001986de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001986fe0 sp=0xc001986f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001986fe8 sp=0xc001986fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 747 [select]: +runtime.gopark(0xc0017d5fb0?, 0x2?, 0xb0?, 0x5d?, 0xc0017d5ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017d5d58 sp=0xc0017d5d38 pc=0x4777ae +runtime.selectgo(0xc0017d5fb0, 0xc0017d5ee0, 0x203a737365726464?, 0x0, 0xa31313635343a74?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017d5ea8 sp=0xc0017d5d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0017d5fe0 sp=0xc0017d5ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017d5fe8 sp=0xc0017d5fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 748 [chan receive]: +runtime.gopark(0xc001963d80?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001963d60 sp=0xc001963d40 pc=0x4777ae +runtime.chanrecv(0xc001713ce0, 0xc001963e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001963dd8 sp=0xc001963d60 pc=0x441325 +runtime.chanrecv1(0x1?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001963e00 sp=0xc001963dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001963fe0 sp=0xc001963e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001963fe8 sp=0xc001963fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 705 [select]: +runtime.gopark(0xc001e16a68?, 0x2?, 0x0?, 0x0?, 0xc001e16a14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e16880 sp=0xc001e16860 pc=0x4777ae +runtime.selectgo(0xc001e16a68, 0xc001e16a10, 0x7f429a447100?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e169d0 sp=0xc001e16880 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc001434000, {0xc001aa6700, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc001e16a98 sp=0xc001e169d0 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001434000, {0xc001aa6700, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001e16b30 sp=0xc001e16a98 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001473ef0, {0xc001aa6700, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001e16b78 sp=0xc001e16b30 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001473ef0}, {0xc001aa6700, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001e16bc0 sp=0xc001e16b78 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000853680, {0xc001aa6700, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001e16c30 sp=0xc001e16bc0 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001aa66f0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001e16d00 sp=0xc001e16c30 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc000853680, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001e16df0 sp=0xc001e16d00 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001aa66f0, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43dd860, 0xc001baacd0}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001e16ea8 sp=0xc001e16df0 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001da50e0, {0x43dd860?, 0xc001baacd0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001e170e0 sp=0xc001e16ea8 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001aa6870, {0x43dd860, 0xc001baacd0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001e171c8 sp=0xc001e170e0 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc001f0fa40, {0x43dd860, 0xc001baacd0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001e17200 sp=0xc001e171c8 pc=0x1fd94b2 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamServer).Recv(0xc00114cfa0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:167 +0x66 fp=0xc001e17240 sp=0xc001e17200 pc=0x2102886 +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream(0xc0010e4930, {0x5b512b8, 0xc00114cfa0}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:87 +0x218 fp=0xc001e17370 sp=0xc001e17240 pc=0x2154718 +github.com/hashicorp/go-plugin/internal/plugin._GRPCBroker_StartStream_Handler({0x41cb860?, 0xc0010e4930}, {0x5b4a6f0?, 0xc001f0fa40}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:148 +0xb6 fp=0xc001e173b0 sp=0xc001e17370 pc=0x2102716 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41cb860, 0xc0010e4930}, {0x5b4ab30, 0xc001aa6870}, 0xc0001a18a8?, 0x47efc98) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001e17400 sp=0xc001e173b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41cb860, 0xc0010e4930}, {0x5b4ab30, 0xc001aa6870}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001e17490 sp=0xc001e17400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41cb860, 0xc0010e4930}, {0x5b492f8, 0xc001da50e0}, 0xc0001a18a8, 0xc001bae0c0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001e177d0 sp=0xc001e17490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41cb860, 0xc0010e4930}, {0x5b492f8, 0xc001da50e0}, 0x3f252a0?, 0xc00114ced0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001e17848 sp=0xc001e177d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000853680, 0xc0010e49c0, 0x770cb80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001e17db0 sp=0xc001e17848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000853680, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001e17f28 sp=0xc001e17db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001e17fe0 sp=0xc001e17f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e17fe8 sp=0xc001e17fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 684 [select]: +runtime.gopark(0xc002423e48?, 0x2?, 0x0?, 0x0?, 0xc002423dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002423c30 sp=0xc002423c10 pc=0x4777ae +runtime.selectgo(0xc002423e48, 0xc002423dc8, 0xc002423dd8?, 0x0, 0xc001968000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002423d80 sp=0xc002423c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0009d6280, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002423e78 sp=0xc002423d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0017da0e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002423f30 sp=0xc002423e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc002423fe0 sp=0xc002423f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002423fe8 sp=0xc002423fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 679 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 685 [select]: +runtime.gopark(0xc001a39f70?, 0x4?, 0x10?, 0x0?, 0xc001a39dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a39c00 sp=0xc001a39be0 pc=0x4777ae +runtime.selectgo(0xc001a39f70, 0xc001a39dd0, 0x40a16a0?, 0x0, 0xc00143a8f0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001a39d50 sp=0xc001a39c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0017d04e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc001a39fc0 sp=0xc001a39d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc001a39fe0 sp=0xc001a39fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a39fe8 sp=0xc001a39fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 679 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 686 [IO wait]: +runtime.gopark(0xc00198ef88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00198ef40 sp=0xc00198ef20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00198ef78 sp=0xc00198ef40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163c2f8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00198ef98 sp=0xc00198ef78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001c9bfa0, 0xc000d98e00?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00198efe0 sp=0xc00198ef98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001c9bf80, {0xc000d98e00, 0x700, 0x700}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00198f0d0 sp=0xc00198efe0 pc=0x50e5a5 +net.(*netFD).Read(0xc001c9bf80, {0xc000d98e00, 0x700, 0x700}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00198f130 sp=0xc00198f0d0 pc=0x89342b +net.(*conn).Read(0xc001d6f240, {0xc000d98e00, 0x700, 0x700}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00198f1b8 sp=0xc00198f130 pc=0x8b176d +net.(*UnixConn).Read(0xc001d6f240, {0xc000d98e00, 0x700, 0x700}) + :1 +0x54 fp=0xc00198f200 sp=0xc00198f1b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0032a2960, {0xc000d98e00, 0x700, 0x700}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc00198f260 sp=0xc00198f200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0019682a8, {0x5b0f6c0, 0xc0032a2960}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc00198f2c8 sp=0xc00198f260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc001968000, {0x5b0f840?, 0xc001d6f240}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc00198f338 sp=0xc00198f2c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc001968000, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc00198f968 sp=0xc00198f338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc001968000, {0xc0017e2000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc00198fa50 sp=0xc00198f968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0007d3060, {0xc0017e2000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc00198fa98 sp=0xc00198fa50 pc=0x10969ed +bufio.(*Reader).Read(0xc0012c8ba0, {0xc000881380, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00198fb58 sp=0xc00198fa98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0012c8ba0}, {0xc000881380, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00198fba0 sp=0xc00198fb58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000881380, 0x9, 0x9}, {0x5b0f0a0, 0xc0012c8ba0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00198fc40 sp=0xc00198fba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000881340) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00198fd78 sp=0xc00198fc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0017d04e0, 0x78?, 0x1769ebc60215b33d?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc00198fec8 sp=0xc00198fd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc0004ef0e0, {0x5b54e00?, 0xc0017d04e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc00198ff80 sp=0xc00198fec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc00198ffe0 sp=0xc00198ff80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00198ffe8 sp=0xc00198ffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 679 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 749 [select]: +runtime.gopark(0xc000d12890?, 0x2?, 0x4e?, 0x1a?, 0xc000d1283c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d126a8 sp=0xc000d12688 pc=0x4777ae +runtime.selectgo(0xc000d12890, 0xc000d12838, 0x7f429a449e00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d127f8 sp=0xc000d126a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0007c08c0, {0xc001aa6970, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc000d128c0 sp=0xc000d127f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0007c08c0, {0xc001aa6970, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc000d12958 sp=0xc000d128c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0012b7ad0, {0xc001aa6970, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc000d129a0 sp=0xc000d12958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0012b7ad0}, {0xc001aa6970, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000d129e8 sp=0xc000d129a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0008ac360, {0xc001aa6970, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc000d12a58 sp=0xc000d129e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001aa6960, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc000d12b28 sp=0xc000d12a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001baebc0?, 0xc0008ac360, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc000d12c18 sp=0xc000d12b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001aa6960, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0014345f0}, 0xc0010e4480?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc000d12cd0 sp=0xc000d12c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001da5e00, {0x43dcf60?, 0xc0014345f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc000d12f08 sp=0xc000d12cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001aa6ab0, {0x43dcf60, 0xc0014345f0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc000d12ff0 sp=0xc000d12f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc001f0faa0, {0x43dcf60, 0xc0014345f0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc000d13028 sp=0xc000d12ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00114d080) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc000d13068 sp=0xc000d13028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00114d080?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc000d130a8 sp=0xc000d13068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00114d090) + :1 +0x46 fp=0xc000d130e0 sp=0xc000d130a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0010e4630, {0x5b51368, 0xc00114d090}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc000d13320 sp=0xc000d130e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00158b250, {0x5b51038, 0xc00114d080}) + :1 +0xaf fp=0xc000d13370 sp=0xc000d13320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00158b250}, {0x5b4a6f0?, 0xc001f0faa0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc000d133b0 sp=0xc000d13370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001aa6ab0}, 0xc0001a1920?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc000d13400 sp=0xc000d133b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001aa6ab0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc000d13490 sp=0xc000d13400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001da5e00}, 0xc0001a1920, 0xc001bae1c0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc000d137d0 sp=0xc000d13490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001da5e00}, 0x3f252a0?, 0xc00114cfb0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc000d13848 sp=0xc000d137d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc0008ac360, 0xc0010e4810, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc000d13db0 sp=0xc000d13848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc0008ac360, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc000d13f28 sp=0xc000d13db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc000d13fe0 sp=0xc000d13f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d13fe8 sp=0xc000d13fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 754 [select]: +runtime.gopark(0xc0017a3fa0?, 0x3?, 0x0?, 0x4e?, 0xc0017a3f3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017a3da8 sp=0xc0017a3d88 pc=0x4777ae +runtime.selectgo(0xc0017a3fa0, 0xc0017a3f34, 0xc0017a3f40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017a3ef8 sp=0xc0017a3da8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:73 +0x1ad fp=0xc0017a3fe0 sp=0xc0017a3ef8 pc=0x2154aad +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017a3fe8 sp=0xc0017a3fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream in goroutine 705 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:71 +0x1d1 + +goroutine 770 [select]: +runtime.gopark(0xc001a07790?, 0x2?, 0x18?, 0x76?, 0xc001a07744?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a075c0 sp=0xc001a075a0 pc=0x4777ae +runtime.selectgo(0xc001a07790, 0xc001a07740, 0x40?, 0x0, 0x40?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001a07710 sp=0xc001a075c0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00165c480) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001a077c0 sp=0xc001a07710 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00184a270, {0x43c4ea0?, 0xc001baec00}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc001a07a20 sp=0xc001a077c0 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0008520b4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001a07a80 sp=0xc001a07a20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc000852000, 0xc0017a4c38, 0xc0017a4c08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc001a07b28 sp=0xc001a07a80 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc000852000, {0x43c4ea0, 0xc001baec00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001a07c68 sp=0xc001a07b28 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc00134ba80, {0x43c4ea0, 0xc001baec00}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc001a07e30 sp=0xc001a07c68 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCStdioStreamStdioClient).Recv(0xc00143aab0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:160 +0x66 fp=0xc001a07e70 sp=0xc001a07e30 pc=0x2103f26 +github.com/hashicorp/go-plugin.(*grpcStdioClient).Run(0xc0007d3440, {0x5b0ee60, 0x779c980}, {0x5b0eb80, 0xc00007c038}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:135 +0x11c fp=0xc001a07f80 sp=0xc001a07e70 pc=0x215c15c +github.com/hashicorp/go-plugin.newGRPCClient.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x6c fp=0xc001a07fe0 sp=0xc001a07f80 pc=0x21598ec +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a07fe8 sp=0xc001a07fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 717 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x84f + +goroutine 771 [select]: +runtime.gopark(0xc0017d4fb0?, 0x2?, 0xb4?, 0x1b?, 0xc0017d4f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017d4de8 sp=0xc0017d4dc8 pc=0x4777ae +runtime.selectgo(0xc0017d4fb0, 0xc0017d4f70, 0x3235202d20202020?, 0x0, 0x202d202020200a38?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017d4f38 sp=0xc0017d4de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0017d4fe0 sp=0xc0017d4f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017d4fe8 sp=0xc0017d4fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 772 [select]: +runtime.gopark(0xc001a35fb0?, 0x2?, 0xb0?, 0x5d?, 0xc001a35ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a35d58 sp=0xc001a35d38 pc=0x4777ae +runtime.selectgo(0xc001a35fb0, 0xc001a35ee0, 0x1128d01?, 0x0, 0x1102359?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001a35ea8 sp=0xc001a35d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001a35fe0 sp=0xc001a35ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a35fe8 sp=0xc001a35fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 773 [chan receive]: +runtime.gopark(0x5059545f54454752?, 0x10444c4549465f45?, 0x4?, 0x12?, 0x455059545f544547?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a36d60 sp=0xc001a36d40 pc=0x4777ae +runtime.chanrecv(0xc00119a120, 0xc001a36e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001a36dd8 sp=0xc001a36d60 pc=0x441325 +runtime.chanrecv1(0x74704f666f656e4f?, 0x140a5812736e6f69?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001a36e00 sp=0xc001a36dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001a36fe0 sp=0xc001a36e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a36fe8 sp=0xc001a36fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 755 [select]: +runtime.gopark(0xc001b11310?, 0x3?, 0xd9?, 0xea?, 0xc001b11292?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b110f0 sp=0xc001b110d0 pc=0x4777ae +runtime.selectgo(0xc001b11310, 0xc001b1128c, 0x46216a2?, 0x0, 0x4?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b11240 sp=0xc001b110f0 pc=0x48916b +github.com/hashicorp/go-plugin.(*grpcStdioServer).StreamStdio(0xc0015108d0, 0x42219c0?, {0x5b4ebb0, 0xc00114d270}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:61 +0x1e5 fp=0xc001b11350 sp=0xc001b11240 pc=0x215bbc5 +github.com/hashicorp/go-plugin/internal/plugin._GRPCStdio_StreamStdio_Handler({0x409e580?, 0xc0015108d0}, {0x5b4a6f0, 0xc001f0fb40}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:185 +0xf7 fp=0xc001b113b0 sp=0xc001b11350 pc=0x2104077 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x409e580, 0xc0015108d0}, {0x5b4ab30, 0xc001cd81b0}, 0xc0004a6048?, 0x47efca8) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001b11400 sp=0xc001b113b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x409e580, 0xc0015108d0}, {0x5b4ab30, 0xc001cd81b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001b11490 sp=0xc001b11400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x409e580, 0xc0015108d0}, {0x5b492f8, 0xc001da5ef0}, 0xc0004a6048, 0xc001baed80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001b117d0 sp=0xc001b11490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x409e580, 0xc0015108d0}, {0x5b492f8, 0xc001da5ef0}, 0x3f252a0?, 0xc00114d190?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001b11848 sp=0xc001b117d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0008539e0, 0xc001267320, 0x770cba0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001b11db0 sp=0xc001b11848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0008539e0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001b11f28 sp=0xc001b11db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001b11fe0 sp=0xc001b11f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b11fe8 sp=0xc001b11fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 750 [select]: +runtime.gopark(0xc001b33fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001b33f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b33de8 sp=0xc001b33dc8 pc=0x4777ae +runtime.selectgo(0xc001b33fb0, 0xc001b33f70, 0xc001713a40?, 0x0, 0xc0012b7110?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b33f38 sp=0xc001b33de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001b33fe0 sp=0xc001b33f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b33fe8 sp=0xc001b33fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 751 [select]: +runtime.gopark(0xc0017a4fb0?, 0x2?, 0xb0?, 0x4d?, 0xc0017a4ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017a4d58 sp=0xc0017a4d38 pc=0x4777ae +runtime.selectgo(0xc0017a4fb0, 0xc0017a4ee0, 0x5655434a55573374?, 0x0, 0x6a55575255575268?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017a4ea8 sp=0xc0017a4d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0017a4fe0 sp=0xc0017a4ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017a4fe8 sp=0xc0017a4fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 752 [chan receive]: +runtime.gopark(0xc001965da8?, 0x1103325?, 0xe0?, 0x87?, 0xc00149cea8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001965d60 sp=0xc001965d40 pc=0x4777ae +runtime.chanrecv(0xc000c9c120, 0xc001965e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001965dd8 sp=0xc001965d60 pc=0x441325 +runtime.chanrecv1(0x1101c91?, 0xc0013219f8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001965e00 sp=0xc001965dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001965fe0 sp=0xc001965e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001965fe8 sp=0xc001965fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 774 [select]: +runtime.gopark(0xc000f0a890?, 0x2?, 0x98?, 0xa7?, 0xc000f0a83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f0a6a8 sp=0xc000f0a688 pc=0x4777ae +runtime.selectgo(0xc000f0a890, 0xc000f0a838, 0x7f429a44c500?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000f0a7f8 sp=0xc000f0a6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000b685f0, {0xc001c610c0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc000f0a8c0 sp=0xc000f0a7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000b685f0, {0xc001c610c0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc000f0a958 sp=0xc000f0a8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001c60e40, {0xc001c610c0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc000f0a9a0 sp=0xc000f0a958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001c60e40}, {0xc001c610c0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000f0a9e8 sp=0xc000f0a9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00165ca20, {0xc001c610c0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc000f0aa58 sp=0xc000f0a9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001c610b0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc000f0ab28 sp=0xc000f0aa58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001e75840?, 0xc00165ca20, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc000f0ac18 sp=0xc000f0ab28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001c610b0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc000b68f00}, 0xc001266cc0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc000f0acd0 sp=0xc000f0ac18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001ec2870, {0x43dcf60?, 0xc000b68f00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc000f0af08 sp=0xc000f0acd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001d42b40, {0x43dcf60, 0xc000b68f00}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc000f0aff0 sp=0xc000f0af08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0007d3680, {0x43dcf60, 0xc000b68f00}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc000f0b028 sp=0xc000f0aff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00143ad60) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc000f0b068 sp=0xc000f0b028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00143ad60?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc000f0b0a8 sp=0xc000f0b068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00143ad70) + :1 +0x46 fp=0xc000f0b0e0 sp=0xc000f0b0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc001266ea0, {0x5b51368, 0xc00143ad70}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc000f0b320 sp=0xc000f0b0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc001510830, {0x5b51038, 0xc00143ad60}) + :1 +0xaf fp=0xc000f0b370 sp=0xc000f0b320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc001510830}, {0x5b4a6f0?, 0xc0007d3680}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc000f0b3b0 sp=0xc000f0b370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001d42b40}, 0xc0014238f0?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc000f0b400 sp=0xc000f0b3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001d42b40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc000f0b490 sp=0xc000f0b400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001ec2870}, 0xc0014238f0, 0xc001e74380) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc000f0b7d0 sp=0xc000f0b490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001ec2870}, 0x3f252a0?, 0xc00143ac90?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc000f0b848 sp=0xc000f0b7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00165ca20, 0xc001266f30, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc000f0bdb0 sp=0xc000f0b848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00165ca20, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc000f0bf28 sp=0xc000f0bdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc000f0bfe0 sp=0xc000f0bf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f0bfe8 sp=0xc000f0bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 753 [select]: +runtime.gopark(0xc001122890?, 0x2?, 0x4e?, 0x1a?, 0xc00112283c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011226a8 sp=0xc001122688 pc=0x4777ae +runtime.selectgo(0xc001122890, 0xc001122838, 0x7f429a44c200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011227f8 sp=0xc0011226a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0007c1180, {0xc001395270, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0011228c0 sp=0xc0011227f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0007c1180, {0xc001395270, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001122958 sp=0xc0011228c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0013951a0, {0xc001395270, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0011229a0 sp=0xc001122958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0013951a0}, {0xc001395270, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0011229e8 sp=0xc0011229a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000a89c20, {0xc001395270, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001122a58 sp=0xc0011229e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001395260, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001122b28 sp=0xc001122a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0012b59c0?, 0xc000a89c20, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001122c18 sp=0xc001122b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001395260, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0007c14a0}, 0x66?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001122cd0 sp=0xc001122c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc00166bd10, {0x43dcf60?, 0xc0007c14a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001122f08 sp=0xc001122cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001395590, {0x43dcf60, 0xc0007c14a0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001122ff0 sp=0xc001122f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0009d24e0, {0x43dcf60, 0xc0007c14a0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001123028 sp=0xc001122ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc0015119a0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001123068 sp=0xc001123028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc0015119a0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0011230a8 sp=0xc001123068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc0015119b0) + :1 +0x46 fp=0xc0011230e0 sp=0xc0011230a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0010e4630, {0x5b51368, 0xc0015119b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001123320 sp=0xc0011230e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00158b250, {0x5b51038, 0xc0015119a0}) + :1 +0xaf fp=0xc001123370 sp=0xc001123320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00158b250}, {0x5b4a6f0?, 0xc0009d24e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0011233b0 sp=0xc001123370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001395590}, 0xc001321c68?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001123400 sp=0xc0011233b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001395590}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001123490 sp=0xc001123400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc00166bd10}, 0xc001321c68, 0xc0012b4d80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0011237d0 sp=0xc001123490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc00166bd10}, 0x3f252a0?, 0xc0015118d0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001123848 sp=0xc0011237d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000a89c20, 0xc0010e4810, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001123db0 sp=0xc001123848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000a89c20, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001123f28 sp=0xc001123db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001123fe0 sp=0xc001123f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001123fe8 sp=0xc001123fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 786 [select]: +runtime.gopark(0xc001a32fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001a32f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a32de8 sp=0xc001a32dc8 pc=0x4777ae +runtime.selectgo(0xc001a32fb0, 0xc001a32f70, 0xc000f40fd0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001a32f38 sp=0xc001a32de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001a32fe0 sp=0xc001a32f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a32fe8 sp=0xc001a32fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 700 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 787 [select]: +runtime.gopark(0xc001b5bfb0?, 0x2?, 0xb0?, 0xbd?, 0xc001b5bee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b5bd58 sp=0xc001b5bd38 pc=0x4777ae +runtime.selectgo(0xc001b5bfb0, 0xc001b5bee0, 0x4aead9?, 0x0, 0x448845?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b5bea8 sp=0xc001b5bd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001b5bfe0 sp=0xc001b5bea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b5bfe8 sp=0xc001b5bfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 700 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 788 [chan receive]: +runtime.gopark(0x20?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b34d60 sp=0xc001b34d40 pc=0x4777ae +runtime.chanrecv(0xc000c6b9e0, 0xc001b34e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b34dd8 sp=0xc001b34d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b34e00 sp=0xc001b34dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001b34fe0 sp=0xc001b34e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b34fe8 sp=0xc001b34fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 700 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 789 [select]: +runtime.gopark(0xc001fb1fa0?, 0x3?, 0xf0?, 0x1d?, 0xc001fb1f3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fb1da8 sp=0xc001fb1d88 pc=0x4777ae +runtime.selectgo(0xc001fb1fa0, 0xc001fb1f34, 0xc001982f40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001fb1ef8 sp=0xc001fb1da8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:184 +0x1b3 fp=0xc001fb1fe0 sp=0xc001fb1ef8 pc=0x21557f3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fb1fe8 sp=0xc001fb1fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream in goroutine 700 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:182 +0x24a + +goroutine 775 [select]: +runtime.gopark(0xc002040a68?, 0x2?, 0x0?, 0x0?, 0xc002040a14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002040880 sp=0xc002040860 pc=0x4777ae +runtime.selectgo(0xc002040a68, 0xc002040a10, 0x7f429a44b300?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020409d0 sp=0xc002040880 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000b69cc0, {0xc001d8f000, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002040a98 sp=0xc0020409d0 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000b69cc0, {0xc001d8f000, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002040b30 sp=0xc002040a98 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001d8e690, {0xc001d8f000, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002040b78 sp=0xc002040b30 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001d8e690}, {0xc001d8f000, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002040bc0 sp=0xc002040b78 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00165cc60, {0xc001d8f000, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002040c30 sp=0xc002040bc0 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001d8eff0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002040d00 sp=0xc002040c30 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc00165cc60, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002040df0 sp=0xc002040d00 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001d8eff0, {0x7f424347ae80, 0x779c980}, 0xc001ec2870?, {0x0, 0x0}, {0x43dd860, 0xc002338050}, 0x12b01b3?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002040ea8 sp=0xc002040df0 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001ec2960, {0x43dd860?, 0xc002338050}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0020410e0 sp=0xc002040ea8 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001d8fe00, {0x43dd860, 0xc002338050}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0020411c8 sp=0xc0020410e0 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0007d3a20, {0x43dd860, 0xc002338050}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002041200 sp=0xc0020411c8 pc=0x1fd94b2 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamServer).Recv(0xc00143b120) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:167 +0x66 fp=0xc002041240 sp=0xc002041200 pc=0x2102886 +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream(0xc001267050, {0x5b512b8, 0xc00143b120}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:87 +0x218 fp=0xc002041370 sp=0xc002041240 pc=0x2154718 +github.com/hashicorp/go-plugin/internal/plugin._GRPCBroker_StartStream_Handler({0x41cb860?, 0xc001267050}, {0x5b4a6f0?, 0xc0007d3a20}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:148 +0xb6 fp=0xc0020413b0 sp=0xc002041370 pc=0x2102716 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41cb860, 0xc001267050}, {0x5b4ab30, 0xc001d8fe00}, 0xc001423e00?, 0x47efc98) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002041400 sp=0xc0020413b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41cb860, 0xc001267050}, {0x5b4ab30, 0xc001d8fe00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002041490 sp=0xc002041400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41cb860, 0xc001267050}, {0x5b492f8, 0xc001ec2960}, 0xc001423e00, 0xc001d3eb40) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0020417d0 sp=0xc002041490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41cb860, 0xc001267050}, {0x5b492f8, 0xc001ec2960}, 0x3f252a0?, 0xc00143b050?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002041848 sp=0xc0020417d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00165cc60, 0xc0012670e0, 0x770cb80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002041db0 sp=0xc002041848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00165cc60, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002041f28 sp=0xc002041db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002041fe0 sp=0xc002041f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002041fe8 sp=0xc002041fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 782 [select]: +runtime.gopark(0xc00120a890?, 0x2?, 0x98?, 0xa7?, 0xc00120a83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00120a6a8 sp=0xc00120a688 pc=0x4777ae +runtime.selectgo(0xc00120a890, 0xc00120a838, 0x7f429a44b000?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00120a7f8 sp=0xc00120a6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc001a0c5a0, {0xc001bba190, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc00120a8c0 sp=0xc00120a7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001a0c5a0, {0xc001bba190, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc00120a958 sp=0xc00120a8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001bba0c0, {0xc001bba190, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc00120a9a0 sp=0xc00120a958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001bba0c0}, {0xc001bba190, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00120a9e8 sp=0xc00120a9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00165d320, {0xc001bba190, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc00120aa58 sp=0xc00120a9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001bba180, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc00120ab28 sp=0xc00120aa58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001d926c0?, 0xc00165d320, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc00120ac18 sp=0xc00120ab28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001bba180, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001a0c8c0}, 0xc001266cc0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc00120acd0 sp=0xc00120ac18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001ec2a50, {0x43dcf60?, 0xc001a0c8c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc00120af08 sp=0xc00120acd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001bba2d0, {0x43dcf60, 0xc001a0c8c0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc00120aff0 sp=0xc00120af08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0007d3b40, {0x43dcf60, 0xc001a0c8c0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc00120b028 sp=0xc00120aff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00143b2c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc00120b068 sp=0xc00120b028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00143b2c0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc00120b0a8 sp=0xc00120b068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00143b2d0) + :1 +0x46 fp=0xc00120b0e0 sp=0xc00120b0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc001266ea0, {0x5b51368, 0xc00143b2d0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc00120b320 sp=0xc00120b0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc001510830, {0x5b51038, 0xc00143b2c0}) + :1 +0xaf fp=0xc00120b370 sp=0xc00120b320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc001510830}, {0x5b4a6f0?, 0xc0007d3b40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc00120b3b0 sp=0xc00120b370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001bba2d0}, 0xc000580060?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc00120b400 sp=0xc00120b3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001bba2d0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc00120b490 sp=0xc00120b400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001ec2a50}, 0xc000580060, 0xc001d3f540) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc00120b7d0 sp=0xc00120b490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001ec2a50}, 0x3f252a0?, 0xc00143b1f0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc00120b848 sp=0xc00120b7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00165d320, 0xc001266f30, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc00120bdb0 sp=0xc00120b848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00165d320, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc00120bf28 sp=0xc00120bdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc00120bfe0 sp=0xc00120bf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00120bfe8 sp=0xc00120bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 777 [select]: +runtime.gopark(0xc001b2ffb0?, 0x2?, 0xb4?, 0x1b?, 0xc001b2ff74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b2fde8 sp=0xc001b2fdc8 pc=0x4777ae +runtime.selectgo(0xc001b2ffb0, 0xc001b2ff70, 0x3235202d20202020?, 0x0, 0x202d202020200a38?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b2ff38 sp=0xc001b2fde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001b2ffe0 sp=0xc001b2ff38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b2ffe8 sp=0xc001b2ffe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 778 [select]: +runtime.gopark(0xc001b30fb0?, 0x2?, 0xb0?, 0xd?, 0xc001b30ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b30d58 sp=0xc001b30d38 pc=0x4777ae +runtime.selectgo(0xc001b30fb0, 0xc001b30ee0, 0x5d53d0?, 0x0, 0x448845?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b30ea8 sp=0xc001b30d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001b30fe0 sp=0xc001b30ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b30fe8 sp=0xc001b30fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 779 [chan receive]: +runtime.gopark(0xb626f746f72702f?, 0x2e7974706d652f66?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a37d60 sp=0xc001a37d40 pc=0x4777ae +runtime.chanrecv(0xc00119ac00, 0xc001a37e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001a37dd8 sp=0xc001a37d60 pc=0x441325 +runtime.chanrecv1(0x28?, 0x14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001a37e00 sp=0xc001a37dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001a37fe0 sp=0xc001a37e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a37fe8 sp=0xc001a37fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 780 [select]: +runtime.gopark(0xc001a38fa0?, 0x3?, 0x0?, 0x4e?, 0xc001a38f3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a38da8 sp=0xc001a38d88 pc=0x4777ae +runtime.selectgo(0xc001a38fa0, 0xc001a38f34, 0xc001a38f40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001a38ef8 sp=0xc001a38da8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:73 +0x1ad fp=0xc001a38fe0 sp=0xc001a38ef8 pc=0x2154aad +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a38fe8 sp=0xc001a38fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream in goroutine 775 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:71 +0x1d1 + +goroutine 760 [select]: +runtime.gopark(0xc001976890?, 0x2?, 0x98?, 0x67?, 0xc00197683c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019766a8 sp=0xc001976688 pc=0x4777ae +runtime.selectgo(0xc001976890, 0xc001976838, 0x7f429a451f00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0019767f8 sp=0xc0019766a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0014353b0, {0xc001e53e70, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0019768c0 sp=0xc0019767f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0014353b0, {0xc001e53e70, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001976958 sp=0xc0019768c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001e53d40, {0xc001e53e70, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0019769a0 sp=0xc001976958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001e53d40}, {0xc001e53e70, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0019769e8 sp=0xc0019769a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0008a2240, {0xc001e53e70, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001976a58 sp=0xc0019769e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001e53e60, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001976b28 sp=0xc001976a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0019f1340?, 0xc0008a2240, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001976c18 sp=0xc001976b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001e53e60, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001435a40}, 0xc001266cc0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001976cd0 sp=0xc001976c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001344a50, {0x43dcf60?, 0xc001435a40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001976f08 sp=0xc001976cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0020861e0, {0x43dcf60, 0xc001435a40}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001976ff0 sp=0xc001976f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc001f0fd80, {0x43dcf60, 0xc001435a40}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001977028 sp=0xc001976ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00114d7e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001977068 sp=0xc001977028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00114d7e0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0019770a8 sp=0xc001977068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00114d7f0) + :1 +0x46 fp=0xc0019770e0 sp=0xc0019770a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc001266ea0, {0x5b51368, 0xc00114d7f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001977320 sp=0xc0019770e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc001510830, {0x5b51038, 0xc00114d7e0}) + :1 +0xaf fp=0xc001977370 sp=0xc001977320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc001510830}, {0x5b4a6f0?, 0xc001f0fd80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0019773b0 sp=0xc001977370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc0020861e0}, 0xc0004a7260?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001977400 sp=0xc0019773b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc0020861e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001977490 sp=0xc001977400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001344a50}, 0xc0004a7260, 0xc0019f0740) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0019777d0 sp=0xc001977490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001344a50}, 0x3f252a0?, 0xc00114d6f0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001977848 sp=0xc0019777d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0008a2240, 0xc001266f30, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001977db0 sp=0xc001977848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0008a2240, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001977f28 sp=0xc001977db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001977fe0 sp=0xc001977f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001977fe8 sp=0xc001977fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 784 [select]: +runtime.gopark(0xc001b31fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001b31f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b31de8 sp=0xc001b31dc8 pc=0x4777ae +runtime.selectgo(0xc001b31fb0, 0xc001b31f70, 0xc0017fac00?, 0x0, 0xc00119af60?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b31f38 sp=0xc001b31de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001b31fe0 sp=0xc001b31f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b31fe8 sp=0xc001b31fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 785 [select]: +runtime.gopark(0xc001b3bfb0?, 0x2?, 0xb0?, 0xbd?, 0xc001b3bee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b3bd58 sp=0xc001b3bd38 pc=0x4777ae +runtime.selectgo(0xc001b3bfb0, 0xc001b3bee0, 0x5d53d0?, 0x0, 0x448845?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b3bea8 sp=0xc001b3bd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001b3bfe0 sp=0xc001b3bea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b3bfe8 sp=0xc001b3bfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 802 [chan receive]: +runtime.gopark(0x3332343637313138?, 0x7d2265686361632f?, 0x2c?, 0x22?, 0x223a22656e69676e?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b2cd60 sp=0xc001b2cd40 pc=0x4777ae +runtime.chanrecv(0xc00119b5c0, 0xc001b2ce60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b2cdd8 sp=0xc001b2cd60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b2ce00 sp=0xc001b2cdd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001b2cfe0 sp=0xc001b2ce00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b2cfe8 sp=0xc001b2cfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 807 [select]: +runtime.gopark(0xc001b2dfb0?, 0x2?, 0xb4?, 0x1b?, 0xc001b2df74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b2dde8 sp=0xc001b2ddc8 pc=0x4777ae +runtime.selectgo(0xc001b2dfb0, 0xc001b2df70, 0x0?, 0x0, 0x202d202020000000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b2df38 sp=0xc001b2dde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001b2dfe0 sp=0xc001b2df38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b2dfe8 sp=0xc001b2dfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 804 [select]: +runtime.gopark(0xc001b4c890?, 0x2?, 0x98?, 0xc7?, 0xc001b4c83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b4c6a8 sp=0xc001b4c688 pc=0x4777ae +runtime.selectgo(0xc001b4c890, 0xc001b4c838, 0x7f429a451000?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b4c7f8 sp=0xc001b4c6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc001a0d040, {0xc001bbb210, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc001b4c8c0 sp=0xc001b4c7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001a0d040, {0xc001bbb210, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001b4c958 sp=0xc001b4c8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001bbadb0, {0xc001bbb210, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001b4c9a0 sp=0xc001b4c958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001bbadb0}, {0xc001bbb210, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001b4c9e8 sp=0xc001b4c9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00165dc20, {0xc001bbb210, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001b4ca58 sp=0xc001b4c9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001bbb200, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001b4cb28 sp=0xc001b4ca58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001a28bc0?, 0xc00165dc20, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001b4cc18 sp=0xc001b4cb28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001bbb200, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001a0d630}, 0xc001266cc0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001b4ccd0 sp=0xc001b4cc18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001ec2b40, {0x43dcf60?, 0xc001a0d630}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001b4cf08 sp=0xc001b4ccd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001bbb350, {0x43dcf60, 0xc001a0d630}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001b4cff0 sp=0xc001b4cf08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc00127c1e0, {0x43dcf60, 0xc001a0d630}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001b4d028 sp=0xc001b4cff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00143b8b0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001b4d068 sp=0xc001b4d028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00143b8b0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc001b4d0a8 sp=0xc001b4d068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00143b8c0) + :1 +0x46 fp=0xc001b4d0e0 sp=0xc001b4d0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc001266ea0, {0x5b51368, 0xc00143b8c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001b4d320 sp=0xc001b4d0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc001510830, {0x5b51038, 0xc00143b8b0}) + :1 +0xaf fp=0xc001b4d370 sp=0xc001b4d320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc001510830}, {0x5b4a6f0?, 0xc00127c1e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc001b4d3b0 sp=0xc001b4d370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001bbb350}, 0xc000581020?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001b4d400 sp=0xc001b4d3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001bbb350}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001b4d490 sp=0xc001b4d400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001ec2b40}, 0xc000581020, 0xc001d93fc0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001b4d7d0 sp=0xc001b4d490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001ec2b40}, 0x3f252a0?, 0xc00143b7e0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001b4d848 sp=0xc001b4d7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00165dc20, 0xc001266f30, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001b4ddb0 sp=0xc001b4d848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00165dc20, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001b4df28 sp=0xc001b4ddb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001b4dfe0 sp=0xc001b4df28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b4dfe8 sp=0xc001b4dfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 808 [select]: +runtime.gopark(0xc001b2efb0?, 0x2?, 0xb0?, 0xed?, 0xc001b2eee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b2ed58 sp=0xc001b2ed38 pc=0x4777ae +runtime.selectgo(0xc001b2efb0, 0xc001b2eee0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b2eea8 sp=0xc001b2ed58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001b2efe0 sp=0xc001b2eea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b2efe8 sp=0xc001b2efe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 809 [chan receive]: +runtime.gopark(0x0?, 0x3?, 0x11?, 0x0?, 0x4db?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b37d60 sp=0xc001b37d40 pc=0x4777ae +runtime.chanrecv(0xc00119bf20, 0xc001b37e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b37dd8 sp=0xc001b37d60 pc=0x441325 +runtime.chanrecv1(0xc?, 0x4dc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b37e00 sp=0xc001b37dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001b37fe0 sp=0xc001b37e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b37fe8 sp=0xc001b37fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1134 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc001ee32b0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00274cdd8 sp=0xc00274cdb8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc000ef5450, 0x2) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc00274ce28 sp=0xc00274cdd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc000ef5440) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc00274ce58 sp=0xc00274ce28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc0020ace00, 0xc001ee32b0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc00274cfb0 sp=0xc00274ce58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc00274cfe0 sp=0xc00274cfb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00274cfe8 sp=0xc00274cfe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 1133 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 811 [select]: +runtime.gopark(0xc001b38fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001b38f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b38de8 sp=0xc001b38dc8 pc=0x4777ae +runtime.selectgo(0xc001b38fb0, 0xc001b38f70, 0xc0017fbaa0?, 0x0, 0xc00119bf80?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b38f38 sp=0xc001b38de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001b38fe0 sp=0xc001b38f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b38fe8 sp=0xc001b38fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 810 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 812 [select]: +runtime.gopark(0xc001b39fb0?, 0x2?, 0x0?, 0x0?, 0xc001b39ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b39d58 sp=0xc001b39d38 pc=0x4777ae +runtime.selectgo(0xc001b39fb0, 0xc001b39ee0, 0x5d53d0?, 0x0, 0x448845?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b39ea8 sp=0xc001b39d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001b39fe0 sp=0xc001b39ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b39fe8 sp=0xc001b39fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 810 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 813 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b3ad60 sp=0xc001b3ad40 pc=0x4777ae +runtime.chanrecv(0xc0014365a0, 0xc001b3ae60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b3add8 sp=0xc001b3ad60 pc=0x441325 +runtime.chanrecv1(0x8?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b3ae00 sp=0xc001b3add8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001b3afe0 sp=0xc001b3ae00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b3afe8 sp=0xc001b3afe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 810 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 790 [select]: +runtime.gopark(0xc001f2a890?, 0x2?, 0x4e?, 0x1a?, 0xc001f2a83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f2a6a8 sp=0xc001f2a688 pc=0x4777ae +runtime.selectgo(0xc001f2a890, 0xc001f2a838, 0x7f429a44fb00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001f2a7f8 sp=0xc001f2a6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00128c8c0, {0xc0013958a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc001f2a8c0 sp=0xc001f2a7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00128c8c0, {0xc0013958a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001f2a958 sp=0xc001f2a8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001924930, {0xc0013958a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001f2a9a0 sp=0xc001f2a958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001924930}, {0xc0013958a0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001f2a9e8 sp=0xc001f2a9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0015f9b00, {0xc0013958a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001f2aa58 sp=0xc001f2a9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001395890, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001f2ab28 sp=0xc001f2aa58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001ab87c0?, 0xc0015f9b00, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001f2ac18 sp=0xc001f2ab28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001395890, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0007c18b0}, 0xc001266cc0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001f2acd0 sp=0xc001f2ac18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc00166be00, {0x43dcf60?, 0xc0007c18b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001f2af08 sp=0xc001f2acd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001395a10, {0x43dcf60, 0xc0007c18b0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001f2aff0 sp=0xc001f2af08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0009d2560, {0x43dcf60, 0xc0007c18b0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001f2b028 sp=0xc001f2aff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc001511ae0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001f2b068 sp=0xc001f2b028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc001511ae0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc001f2b0a8 sp=0xc001f2b068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc001511af0) + :1 +0x46 fp=0xc001f2b0e0 sp=0xc001f2b0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc001266ea0, {0x5b51368, 0xc001511af0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001f2b320 sp=0xc001f2b0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc001510830, {0x5b51038, 0xc001511ae0}) + :1 +0xaf fp=0xc001f2b370 sp=0xc001f2b320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc001510830}, {0x5b4a6f0?, 0xc0009d2560}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc001f2b3b0 sp=0xc001f2b370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001395a10}, 0xc001321e00?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001f2b400 sp=0xc001f2b3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001395a10}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001f2b490 sp=0xc001f2b400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc00166be00}, 0xc001321e00, 0xc0012b5a80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001f2b7d0 sp=0xc001f2b490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc00166be00}, 0x3f252a0?, 0xc001511a10?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001f2b848 sp=0xc001f2b7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0015f9b00, 0xc001266f30, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001f2bdb0 sp=0xc001f2b848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0015f9b00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001f2bf28 sp=0xc001f2bdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001f2bfe0 sp=0xc001f2bf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f2bfe8 sp=0xc001f2bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 757 [select]: +runtime.gopark(0xc001a33fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001a33f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a33de8 sp=0xc001a33dc8 pc=0x4777ae +runtime.selectgo(0xc001a33fb0, 0xc001a33f70, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001a33f38 sp=0xc001a33de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001a33fe0 sp=0xc001a33f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a33fe8 sp=0xc001a33fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 765 [chan receive]: +runtime.gopark(0xc000b3d020?, 0x27?, 0x0?, 0x0?, 0x460fed5?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b57ef8 sp=0xc001b57ed8 pc=0x4777ae +runtime.chanrecv(0xc001143200, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b57f70 sp=0xc001b57ef8 pc=0x441325 +runtime.chanrecv1(0xc001fefae0?, 0x47e975?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b57f98 sp=0xc001b57f70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc001b57fe0 sp=0xc001b57f98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b57fe8 sp=0xc001b57fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 763 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 820 [select]: +runtime.gopark(0xc001bca890?, 0x2?, 0x98?, 0xa7?, 0xc001bca83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bca6a8 sp=0xc001bca688 pc=0x4777ae +runtime.selectgo(0xc001bca890, 0xc001bca838, 0x7f429a454c00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001bca7f8 sp=0xc001bca6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0007c1e50, {0xc001dcf360, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc001bca8c0 sp=0xc001bca7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0007c1e50, {0xc001dcf360, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001bca958 sp=0xc001bca8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001dcebd0, {0xc001dcf360, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001bca9a0 sp=0xc001bca958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001dcebd0}, {0xc001dcf360, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001bca9e8 sp=0xc001bca9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000e50900, {0xc001dcf360, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001bcaa58 sp=0xc001bca9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001dcf350, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001bcab28 sp=0xc001bcaa58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001ab9680?, 0xc000e50900, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001bcac18 sp=0xc001bcab28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001dcf350, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001aba4b0}, 0xc0010e4480?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001bcacd0 sp=0xc001bcac18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc00166bef0, {0x43dcf60?, 0xc001aba4b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001bcaf08 sp=0xc001bcacd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001dcf4a0, {0x43dcf60, 0xc001aba4b0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001bcaff0 sp=0xc001bcaf08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0009d2f40, {0x43dcf60, 0xc001aba4b0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001bcb028 sp=0xc001bcaff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc001511f20) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001bcb068 sp=0xc001bcb028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc001511f20?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc001bcb0a8 sp=0xc001bcb068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc001511f30) + :1 +0x46 fp=0xc001bcb0e0 sp=0xc001bcb0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0010e4630, {0x5b51368, 0xc001511f30}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001bcb320 sp=0xc001bcb0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00158b250, {0x5b51038, 0xc001511f20}) + :1 +0xaf fp=0xc001bcb370 sp=0xc001bcb320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00158b250}, {0x5b4a6f0?, 0xc0009d2f40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc001bcb3b0 sp=0xc001bcb370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001dcf4a0}, 0xc0012e4390?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001bcb400 sp=0xc001bcb3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001dcf4a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001bcb490 sp=0xc001bcb400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc00166bef0}, 0xc0012e4390, 0xc001ab8c80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001bcb7d0 sp=0xc001bcb490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc00166bef0}, 0x3f252a0?, 0xc001511e50?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001bcb848 sp=0xc001bcb7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000e50900, 0xc0010e4810, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001bcbdb0 sp=0xc001bcb848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000e50900, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001bcbf28 sp=0xc001bcbdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001bcbfe0 sp=0xc001bcbf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bcbfe8 sp=0xc001bcbfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 801 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x20?, 0x3?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001997cd0 sp=0xc001997cb0 pc=0x4777ae +runtime.chanrecv(0xc00197e8a0, 0xc001997de0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001997d48 sp=0xc001997cd0 pc=0x441325 +runtime.chanrecv1(0xc001997df0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001997d70 sp=0xc001997d48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc001997f30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc001997e00 sp=0xc001997d70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc001fefdb0, 0x420c60?, 0xc001924e70) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc001997fa0 sp=0xc001997e00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc001997fe0 sp=0xc001997fa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001997fe8 sp=0xc001997fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 798 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 763 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0xc3?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b13cd0 sp=0xc001b13cb0 pc=0x4777ae +runtime.chanrecv(0xc001c187e0, 0xc001b13de0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b13d48 sp=0xc001b13cd0 pc=0x441325 +runtime.chanrecv1(0xc001b13df0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b13d70 sp=0xc001b13d48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc001b13f30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc001b13e00 sp=0xc001b13d70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc001fefdb0, 0x0?, 0xc001c163c0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc001b13fa0 sp=0xc001b13e00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc001b13fe0 sp=0xc001b13fa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b13fe8 sp=0xc001b13fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 762 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 1378 [select]: +runtime.gopark(0xc0024d1f70?, 0x2?, 0xb4?, 0x1b?, 0xc0024d1ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024d1d20 sp=0xc0024d1d00 pc=0x4777ae +runtime.selectgo(0xc0024d1f70, 0xc0024d1eb8, 0x1fa3b79?, 0x0, 0x5b50208?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0024d1e70 sp=0xc0024d1d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001d86a80, {0x5b426e0, 0xc002338c80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0024d1fa0 sp=0xc0024d1e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0024d1fe0 sp=0xc0024d1fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024d1fe8 sp=0xc0024d1fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1361 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1132 [select]: +runtime.gopark(0xc002057fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002057f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002057de8 sp=0xc002057dc8 pc=0x4777ae +runtime.selectgo(0xc002057fb0, 0xc002057f70, 0x4aead9?, 0x0, 0x23220cf?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002057f38 sp=0xc002057de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002057fe0 sp=0xc002057f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002057fe8 sp=0xc002057fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1138 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 794 [select]: +runtime.gopark(0xc001b6dfb0?, 0x2?, 0xb4?, 0x1b?, 0xc001b6df74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b6dde8 sp=0xc001b6ddc8 pc=0x4777ae +runtime.selectgo(0xc001b6dfb0, 0xc001b6df70, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b6df38 sp=0xc001b6dde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001b6dfe0 sp=0xc001b6df38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b6dfe8 sp=0xc001b6dfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 795 [select]: +runtime.gopark(0xc001b54fb0?, 0x2?, 0xb0?, 0x4d?, 0xc001b54ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b54d58 sp=0xc001b54d38 pc=0x4777ae +runtime.selectgo(0xc001b54fb0, 0xc001b54ee0, 0xe?, 0x0, 0x8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b54ea8 sp=0xc001b54d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001b54fe0 sp=0xc001b54ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b54fe8 sp=0xc001b54fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 796 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b55d60 sp=0xc001b55d40 pc=0x4777ae +runtime.chanrecv(0xc001088060, 0xc001b55e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b55dd8 sp=0xc001b55d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b55e00 sp=0xc001b55dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001b55fe0 sp=0xc001b55e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b55fe8 sp=0xc001b55fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 834 [select]: +runtime.gopark(0xc001b7b448?, 0x2?, 0xd0?, 0xb2?, 0xc001b7b3fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b7b278 sp=0xc001b7b258 pc=0x4777ae +runtime.selectgo(0xc001b7b448, 0xc001b7b3f8, 0xc0015113b8?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b7b3c8 sp=0xc001b7b278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00094f0e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001b7b478 sp=0xc001b7b3c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00194c750, {0x42219c0?, 0xc001924f00}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc001b7b6d8 sp=0xc001b7b478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00094e414?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001b7b738 sp=0xc001b7b6d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00094e360, 0xc001b7b8f0, 0xc001b7b8c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc001b7b7e0 sp=0xc001b7b738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00094e360, {0x42219c0, 0xc001924f00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001b7b920 sp=0xc001b7b7e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc001924fc0}, {0x4661da0, 0x1f}, {0x4278de0, 0xc001924ed0}, {0x42219c0, 0xc001924f00}, 0x0?, {0xc001f0f380, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc001b7b9c0 sp=0xc001b7b920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc001fefae0}, {0x4661da0, 0x1f}, {0x4278de0, 0xc001924ed0}, {0x42219c0, 0xc001924f00}, 0xc001838400, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc001b7bd30 sp=0xc001b7b9c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc001838400, {0x5b426e0, 0xc001fefae0}, {0x4661da0, 0x1f}, {0x4278de0, 0xc001924ed0}, {0x42219c0, 0xc001924f00}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc001b7be58 sp=0xc001b7bd30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseManagementAPI(0xc00158bb70, {0x5b426e0, 0xc001fefae0}, 0x0?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:51 +0xee fp=0xc001b7bee8 sp=0xc001b7be58 pc=0x21ab66e +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeManagementAPI.func2(0x1) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:190 +0xf2 fp=0xc001b7bf58 sp=0xc001b7bee8 pc=0x21a7f32 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc001b7bfe0 sp=0xc001b7bf58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b7bfe8 sp=0xc001b7bfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 798 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 798 [select]: +runtime.gopark(0xc001b56f28?, 0x2?, 0xb4?, 0x1b?, 0xc001b56ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b56d30 sp=0xc001b56d10 pc=0x4777ae +runtime.selectgo(0xc001b56f28, 0xc001b56eb8, 0x0?, 0x0, 0xc001b56f20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b56e80 sp=0xc001b56d30 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc001924a80, 0xc000f1ff08, 0xc00158bd10) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001b56f58 sp=0xc001b56e80 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeManagementAPI(0xc001924a80, {0x5b6f770?, 0xc0010c0e70}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:185 +0x10a fp=0xc001b56f98 sp=0xc001b56f58 pc=0x21a7e0a +github.com/rancher/opni/pkg/management.NewServer.func1.2() + /home/ec2-user/opni/pkg/management/server.go:156 +0x62 fp=0xc001b56fe0 sp=0xc001b56f98 pc=0x231c0c2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b56fe8 sp=0xc001b56fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.NewServer.func1 in goroutine 797 + /home/ec2-user/opni/pkg/management/server.go:156 +0x137 + +goroutine 799 [select]: +runtime.gopark(0xc001b58f28?, 0x2?, 0xb4?, 0x1b?, 0xc001b58ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b58d30 sp=0xc001b58d10 pc=0x4777ae +runtime.selectgo(0xc001b58f28, 0xc001b58eb8, 0x22c21f5?, 0x0, 0xc001b58f20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b58e80 sp=0xc001b58d30 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc001924a80, 0xc000581950, 0xc00143bd30) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001b58f58 sp=0xc001b58e80 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeNodeManagerServer(0xc001924a80, {0x5b21b70?, 0xc0017e1740}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:198 +0x10a fp=0xc001b58f98 sp=0xc001b58f58 pc=0x21a810a +github.com/rancher/opni/pkg/management.NewServer.func1.3() + /home/ec2-user/opni/pkg/management/server.go:158 +0x62 fp=0xc001b58fe0 sp=0xc001b58f98 pc=0x231c022 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b58fe8 sp=0xc001b58fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.NewServer.func1 in goroutine 797 + /home/ec2-user/opni/pkg/management/server.go:158 +0x245 + +goroutine 1309 [IO wait]: +runtime.gopark(0xc0024d6fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024d6f98 sp=0xc0024d6f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0024d6fd0 sp=0xc0024d6f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169d580, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0024d6ff0 sp=0xc0024d6fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0025f2020, 0xc002d73c00?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0024d7038 sp=0xc0024d6ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0025f2000, {0xc002d73c00, 0xc00, 0xc00}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0024d7128 sp=0xc0024d7038 pc=0x50e5a5 +net.(*netFD).Read(0xc0025f2000, {0xc002d73c00, 0xc00, 0xc00}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0024d7188 sp=0xc0024d7128 pc=0x89342b +net.(*conn).Read(0xc001d6efb8, {0xc002d73c00, 0xc00, 0xc00}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0024d7210 sp=0xc0024d7188 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6efb8, {0xc002d73c00, 0xc00, 0xc00}) + :1 +0x54 fp=0xc0024d7258 sp=0xc0024d7210 pc=0x8d72f4 +crypto/tls.(*atLeastReader).Read(0xc00304a2b8, {0xc002d73c00, 0xc00, 0xc00}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0024d72b8 sp=0xc0024d7258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc00133a2a8, {0x5b0f6c0, 0xc00304a2b8}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0024d7320 sp=0xc0024d72b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc00133a000, {0x5b0f820?, 0xc001d6efb8}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc0024d7390 sp=0xc0024d7320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc00133a000, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0024d79c0 sp=0xc0024d7390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc00133a000, {0xc00265c000, 0x8000, 0xc0024d7af8?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc0024d7aa8 sp=0xc0024d79c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0015ff7c0, {0xc00265c000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc0024d7af0 sp=0xc0024d7aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc002081980, {0xc000b9a740, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0024d7bb0 sp=0xc0024d7af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc002081980}, {0xc000b9a740, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0024d7bf8 sp=0xc0024d7bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000b9a740, 0x9, 0x9}, {0x5b0f0a0, 0xc002081980}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0024d7c98 sp=0xc0024d7bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000b9a700) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0024d7dd0 sp=0xc0024d7c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000adcfc0, 0xc0018534a0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0024d7fb0 sp=0xc0024d7dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0024d7fe0 sp=0xc0024d7fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024d7fe8 sp=0xc0024d7fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1353 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 758 [select]: +runtime.gopark(0xc001b69fb0?, 0x2?, 0xb0?, 0x9d?, 0xc001b69ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b69d58 sp=0xc001b69d38 pc=0x4777ae +runtime.selectgo(0xc001b69fb0, 0xc001b69ee0, 0x69747265706f7270?, 0x0, 0x448845?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b69ea8 sp=0xc001b69d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001b69fe0 sp=0xc001b69ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b69fe8 sp=0xc001b69fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 759 [chan receive]: +runtime.gopark(0x632f0a1a2a013a0f?, 0x6572756769666e6f?, 0x12?, 0x72?, 0x6e6f437472656c41?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a34d60 sp=0xc001a34d40 pc=0x4777ae +runtime.chanrecv(0xc001678780, 0xc001a34e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001a34dd8 sp=0xc001a34d60 pc=0x441325 +runtime.chanrecv1(0x2e676e697472656c?, 0x6f697469646e6f43?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001a34e00 sp=0xc001a34dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001a34fe0 sp=0xc001a34e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a34fe8 sp=0xc001a34fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 815 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0xe0?, 0x6?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b0fcd0 sp=0xc001b0fcb0 pc=0x4777ae +runtime.chanrecv(0xc00197e9c0, 0xc001b0fde0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b0fd48 sp=0xc001b0fcd0 pc=0x441325 +runtime.chanrecv1(0xc001b0fdf0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b0fd70 sp=0xc001b0fd48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc001b0ff30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc001b0fe00 sp=0xc001b0fd70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc001fefdb0, 0x2?, 0xc001b525d0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc001b0ffa0 sp=0xc001b0fe00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc001b0ffe0 sp=0xc001b0ffa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b0ffe8 sp=0xc001b0ffe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 799 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 816 [select]: +runtime.gopark(0xc001b77448?, 0x2?, 0xd0?, 0x72?, 0xc001b773fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b77278 sp=0xc001b77258 pc=0x4777ae +runtime.selectgo(0xc001b77448, 0xc001b773f8, 0xc0015113b8?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b773c8 sp=0xc001b77278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00110efc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001b77478 sp=0xc001b773c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc0017fdad0, {0x42219c0?, 0xc001b52a50}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc001b776d8 sp=0xc001b77478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00110ead4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001b77738 sp=0xc001b776d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00110ea20, 0xc001b778f0, 0xc001b778c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc001b777e0 sp=0xc001b77738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00110ea20, {0x42219c0, 0xc001b52a50}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001b77920 sp=0xc001b777e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc001b52b10}, {0x4673e80, 0x23}, {0x4278de0, 0xc001b52a20}, {0x42219c0, 0xc001b52a50}, 0x0?, {0xc001f0f380, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc001b779c0 sp=0xc001b77920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc001fefae0}, {0x4673e80, 0x23}, {0x4278de0, 0xc001b52a20}, {0x42219c0, 0xc001b52a50}, 0xc001838400, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc001b77d30 sp=0xc001b779c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc001838400, {0x5b426e0, 0xc001fefae0}, {0x4673e80, 0x23}, {0x4278de0, 0xc001b52a20}, {0x42219c0, 0xc001b52a50}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc001b77e58 sp=0xc001b77d30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseNodeManagerClient(0xc00158bb70, {0x5b426e0, 0xc001fefae0}, 0xc001707750?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:60 +0xee fp=0xc001b77ee8 sp=0xc001b77e58 pc=0x21ab7ee +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeNodeManagerServer.func2(0x2) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:203 +0xf2 fp=0xc001b77f58 sp=0xc001b77ee8 pc=0x21a8232 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc001b77fe0 sp=0xc001b77f58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b77fe8 sp=0xc001b77fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 799 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 835 [chan receive]: +runtime.gopark(0x22c2125?, 0xc000081750?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b35ef8 sp=0xc001b35ed8 pc=0x4777ae +runtime.chanrecv(0xc001143200, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b35f70 sp=0xc001b35ef8 pc=0x441325 +runtime.chanrecv1(0xc001fefae0?, 0x5b6f770?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b35f98 sp=0xc001b35f70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc001b35fe0 sp=0xc001b35f98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b35fe8 sp=0xc001b35fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 801 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 836 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bd7978 sp=0xc001bd7958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001bd79b0 sp=0xc001bd7978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163c200, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001bd79d0 sp=0xc001bd79b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00193ab20, 0xc001bd7a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001bd7a18 sp=0xc001bd79d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc00193ab00) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001bd7b08 sp=0xc001bd7a18 pc=0x513f65 +net.(*netFD).accept(0xc00193ab00) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001bd7c00 sp=0xc001bd7b08 pc=0x8967be +net.(*UnixListener).accept(0xc001924ea0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc001bd7c40 sp=0xc001bd7c00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc001924ea0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc001bd7cb0 sp=0xc001bd7c40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc001dedf00) + :1 +0x44 fp=0xc001bd7ce8 sp=0xc001bd7cb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc00191c780, {0x5b2c6a0, 0xc001dedf00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc001bd7f18 sp=0xc001bd7ce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc001bd7f68 sp=0xc001bd7f18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc000b30200?, 0xc00158bf40?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001bd7fa8 sp=0xc001bd7f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001bd7fe0 sp=0xc001bd7fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bd7fe8 sp=0xc001bd7fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 801 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 837 [select]: +runtime.gopark(0xc001962f38?, 0x2?, 0x69?, 0x97?, 0xc001962f14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001962d90 sp=0xc001962d70 pc=0x4777ae +runtime.selectgo(0xc001962f38, 0xc001962f10, 0xc001962f18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001962ee0 sp=0xc001962d90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc001962f68 sp=0xc001962ee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc000126198?, 0xc0018be000?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001962fa8 sp=0xc001962f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001962fe0 sp=0xc001962fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001962fe8 sp=0xc001962fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 801 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 838 [chan receive]: +runtime.gopark(0x0?, 0x2?, 0x1?, 0xd6?, 0xc000086768?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d4bef8 sp=0xc001d4bed8 pc=0x4777ae +runtime.chanrecv(0xc001143200, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001d4bf70 sp=0xc001d4bef8 pc=0x441325 +runtime.chanrecv1(0xc001fefae0?, 0x2106c90?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001d4bf98 sp=0xc001d4bf70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc001d4bfe0 sp=0xc001d4bf98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d4bfe8 sp=0xc001d4bfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 815 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 839 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bd9978 sp=0xc001bd9958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001bd99b0 sp=0xc001bd9978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163c108, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001bd99d0 sp=0xc001bd99b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0017cf1a0, 0xc001bd9a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001bd9a18 sp=0xc001bd99d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0017cf180) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001bd9b08 sp=0xc001bd9a18 pc=0x513f65 +net.(*netFD).accept(0xc0017cf180) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001bd9c00 sp=0xc001bd9b08 pc=0x8967be +net.(*UnixListener).accept(0xc001b529f0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc001bd9c40 sp=0xc001bd9c00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc001b529f0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc001bd9cb0 sp=0xc001bd9c40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc00127c6a0) + :1 +0x44 fp=0xc001bd9ce8 sp=0xc001bd9cb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc00191c960, {0x5b2c6a0, 0xc00127c6a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc001bd9f18 sp=0xc001bd9ce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc001bd9f68 sp=0xc001bd9f18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc000b30620?, 0xc0018be140?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001bd9fa8 sp=0xc001bd9f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001bd9fe0 sp=0xc001bd9fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bd9fe8 sp=0xc001bd9fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 815 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 840 [select]: +runtime.gopark(0xc001b36f38?, 0x2?, 0x2?, 0x0?, 0xc001b36f14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b36d90 sp=0xc001b36d70 pc=0x4777ae +runtime.selectgo(0xc001b36f38, 0xc001b36f10, 0xc001b36f18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b36ee0 sp=0xc001b36d90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc001b36f68 sp=0xc001b36ee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc000126258?, 0xc0018be160?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001b36fa8 sp=0xc001b36f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001b36fe0 sp=0xc001b36fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b36fe8 sp=0xc001b36fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 815 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 823 [chan receive]: +runtime.gopark(0xc001ded584?, 0x4be6c0?, 0x80?, 0xd5?, 0x4be720?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00272ecf8 sp=0xc00272ecd8 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00272ed70 sp=0xc00272ecf8 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0xc00290dbf0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00272ed98 sp=0xc00272ed70 pc=0x440f72 +github.com/rancher/opni/plugins/metrics/pkg/gateway.(*Plugin).UseManagementAPI(0xc000379180, {0x5b6f168, 0xc001854f90}) + /home/ec2-user/opni/plugins/metrics/pkg/gateway/system.go:69 +0x67c fp=0xc00272efa0 sp=0xc00272ed98 pc=0x3db3f9c +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseManagementAPI(0xc001160340, {0x4b05a9?, 0xc001b0d0f0?}, 0xc002087aa0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:102 +0x30f fp=0xc00272f098 sp=0xc00272efa0 pc=0x21a640f +github.com/rancher/opni/pkg/plugins/apis/system._System_UseManagementAPI_Handler.func1({0x5b426a8, 0xc001c16000}, {0x4278de0?, 0xc002087aa0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:151 +0x83 fp=0xc00272f0f0 sp=0xc00272f098 pc=0x21ac063 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc001c16000}, {0x4278de0, 0xc002087aa0}, 0xc001f0fe00, 0xc0004a78c0) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc00272f148 sp=0xc00272f0f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc001c16000}, {0x4278de0, 0xc002087aa0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc00272f1e8 sp=0xc00272f148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc002087680}, {0x4278de0, 0xc002087aa0}, 0xc001f0fe00, 0xc0019f1480) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc00272f6b8 sp=0xc00272f1e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc002087680}, {0x4278de0, 0xc002087aa0}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc00272f740 sp=0xc00272f6b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseManagementAPI_Handler({0x4278ea0?, 0xc001160340}, {0x5b426a8, 0xc002087680}, 0xc002012f50, 0xc001ded8a0) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:153 +0x1c4 fp=0xc00272f7d0 sp=0xc00272f740 pc=0x21abec4 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000e510e0, 0xc0010e57d0, 0x771eca0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc00272fdb0 sp=0xc00272f7d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000e510e0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc00272ff28 sp=0xc00272fdb0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc00272ffe0 sp=0xc00272ff28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00272ffe8 sp=0xc00272ffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 824 [chan receive]: +runtime.gopark(0x4be6c0?, 0xc0014482d0?, 0x20?, 0xe7?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e36f08 sp=0xc001e36ee8 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001e36f80 sp=0xc001e36f08 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0x5b11a20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001e36fa8 sp=0xc001e36f80 pc=0x440f72 +github.com/rancher/opni/plugins/metrics/pkg/gateway.(*Plugin).UseNodeManagerClient(0xc000379180, {0x5b11a20, 0xc0018552b0}) + /home/ec2-user/opni/plugins/metrics/pkg/gateway/system.go:74 +0x93 fp=0xc001e36fd0 sp=0xc001e36fa8 pc=0x3db4593 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseNodeManagerClient(0xc001160340, {0x4b05a9?, 0xc001a0b0f0?}, 0xc001ac23c0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:113 +0x18d fp=0xc001e37098 sp=0xc001e36fd0 pc=0x21a68cd +github.com/rancher/opni/pkg/plugins/apis/system._System_UseNodeManagerClient_Handler.func1({0x5b426a8, 0xc001ac2480}, {0x4278de0?, 0xc001ac23c0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:169 +0x83 fp=0xc001e370f0 sp=0xc001e37098 pc=0x21ac443 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc001ac2480}, {0x4278de0, 0xc001ac23c0}, 0xc0009d3240, 0xc0012e46d8) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc001e37148 sp=0xc001e370f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc001ac2480}, {0x4278de0, 0xc001ac23c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc001e371e8 sp=0xc001e37148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc001ac2390}, {0x4278de0, 0xc001ac23c0}, 0xc0009d3240, 0xc001ab9840) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc001e376b8 sp=0xc001e371e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc001ac2390}, {0x4278de0, 0xc001ac23c0}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc001e37740 sp=0xc001e376b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseNodeManagerClient_Handler({0x4278ea0?, 0xc001160340}, {0x5b426a8, 0xc001ac2390}, 0xc0018b8380, 0xc001ded8a0) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:171 +0x1c4 fp=0xc001e377d0 sp=0xc001e37740 pc=0x21ac2a4 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000e51200, 0xc0010e57d0, 0x771ecb8, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc001e37db0 sp=0xc001e377d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000e51200, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc001e37f28 sp=0xc001e37db0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001e37fe0 sp=0xc001e37f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e37fe8 sp=0xc001e37fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 850 [select]: +runtime.gopark(0xc001b32fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001b32f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b32de8 sp=0xc001b32dc8 pc=0x4777ae +runtime.selectgo(0xc001b32fb0, 0xc001b32f70, 0xc0017077d0?, 0x0, 0xc000000002?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b32f38 sp=0xc001b32de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001b32fe0 sp=0xc001b32f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b32fe8 sp=0xc001b32fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 851 [select]: +runtime.gopark(0xc001b5afb0?, 0x2?, 0xb0?, 0xad?, 0xc001b5aee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b5ad58 sp=0xc001b5ad38 pc=0x4777ae +runtime.selectgo(0xc001b5afb0, 0xc001b5aee0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b5aea8 sp=0xc001b5ad58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001b5afe0 sp=0xc001b5aea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b5afe8 sp=0xc001b5afe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 852 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d47d60 sp=0xc001d47d40 pc=0x4777ae +runtime.chanrecv(0xc0014374a0, 0xc001d47e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001d47dd8 sp=0xc001d47d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001d47e00 sp=0xc001d47dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001d47fe0 sp=0xc001d47e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d47fe8 sp=0xc001d47fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 825 [select]: +runtime.gopark(0xc001bddf70?, 0x2?, 0xb4?, 0x1b?, 0xc001bddebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bddd20 sp=0xc001bddd00 pc=0x4777ae +runtime.selectgo(0xc001bddf70, 0xc001bddeb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001bdde70 sp=0xc001bddd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0009d39e0, {0x5b426e0, 0xc001aba9b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001bddfa0 sp=0xc001bdde70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001bddfe0 sp=0xc001bddfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bddfe8 sp=0xc001bddfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 822 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 826 [select]: +runtime.gopark(0xc001bdbf70?, 0x2?, 0xb4?, 0x1b?, 0xc001bdbebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bdbd20 sp=0xc001bdbd00 pc=0x4777ae +runtime.selectgo(0xc001bdbf70, 0xc001bdbeb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001bdbe70 sp=0xc001bdbd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0009d3a60, {0x5b426e0, 0xc001abaa00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001bdbfa0 sp=0xc001bdbe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001bdbfe0 sp=0xc001bdbfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bdbfe8 sp=0xc001bdbfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 822 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1056 [chan receive]: +runtime.gopark(0x4aef8c?, 0x4b0a55?, 0x28?, 0x8a?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001984e18 sp=0xc001984df8 pc=0x4777ae +runtime.chanrecv(0xc00235d680, 0xc001984ed0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001984e90 sp=0xc001984e18 pc=0x441325 +runtime.chanrecv2(0xc001658a28?, 0x4be3ae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc001984eb8 sp=0xc001984e90 pc=0x440f92 +github.com/nats-io/nats%2ego.(*Conn).flusher(0xc001658a00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:3324 +0x1a5 fp=0xc001984fc0 sp=0xc001984eb8 pc=0x1f792e5 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x34 fp=0xc001984fe0 sp=0xc001984fc0 pc=0x1f714b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001984fe8 sp=0xc001984fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 823 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x4cb + +goroutine 1148 [IO wait]: +runtime.gopark(0xc0029d3650?, 0x4d1aca?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029d35e0 sp=0xc0029d35c0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0029d3618 sp=0xc0029d35e0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494240, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0029d3638 sp=0xc0029d3618 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0027d4720, 0xc000dee000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0029d3680 sp=0xc0029d3638 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0027d4700, {0xc000dee000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0029d3770 sp=0xc0029d3680 pc=0x50e5a5 +net.(*netFD).Read(0xc0027d4700, {0xc000dee000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0029d37d0 sp=0xc0029d3770 pc=0x89342b +net.(*conn).Read(0xc000bdfcc8, {0xc000dee000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0029d3858 sp=0xc0029d37d0 pc=0x8b176d +net.(*TCPConn).Read(0xc000bdfcc8, {0xc000dee000, 0x1000, 0x1000}) + :1 +0x54 fp=0xc0029d38a0 sp=0xc0029d3858 pc=0x8d72f4 +net/http.(*connReader).Read(0xc0021ba1b0, {0xc000dee000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:791 +0x2b5 fp=0xc0029d3938 sp=0xc0029d38a0 pc=0xa68115 +bufio.(*Reader).fill(0xc000d64360) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:113 +0x29a fp=0xc0029d39c8 sp=0xc0029d3938 pc=0x5f863a +bufio.(*Reader).Peek(0xc000d64360, 0x4) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:151 +0xc7 fp=0xc0029d3a50 sp=0xc0029d39c8 pc=0x5f88e7 +net/http.(*conn).serve(0xc001172360, {0x5b426a8, 0xc0020083f0}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:2044 +0xe7c fp=0xc0029d3fa0 sp=0xc0029d3a50 pc=0xa7221c +net/http.(*Server).Serve.func3() + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3086 +0x50 fp=0xc0029d3fe0 sp=0xc0029d3fa0 pc=0xa78af0 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029d3fe8 sp=0xc0029d3fe0 pc=0x4ad421 +created by net/http.(*Server).Serve in goroutine 1161 + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3086 +0x80d + +goroutine 829 [select]: +runtime.gopark(0xc001a0bf70?, 0x2?, 0xb4?, 0x1b?, 0xc001a0bebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a0bd20 sp=0xc001a0bd00 pc=0x4777ae +runtime.selectgo(0xc001a0bf70, 0xc001a0beb8, 0x4aead9?, 0x0, 0x215774d?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001a0be70 sp=0xc001a0bd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0009d3ee0, {0x5b426e0, 0xc001abac30}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001a0bfa0 sp=0xc001a0be70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001a0bfe0 sp=0xc001a0bfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a0bfe8 sp=0xc001a0bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 823 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 830 [select]: +runtime.gopark(0xc001b0df70?, 0x2?, 0xb4?, 0x1b?, 0xc001b0debc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b0dd20 sp=0xc001b0dd00 pc=0x4777ae +runtime.selectgo(0xc001b0df70, 0xc001b0deb8, 0x4aead9?, 0x0, 0x215774d?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b0de70 sp=0xc001b0dd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0009d3f60, {0x5b426e0, 0xc001abac80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001b0dfa0 sp=0xc001b0de70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001b0dfe0 sp=0xc001b0dfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b0dfe8 sp=0xc001b0dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 823 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 906 [chan receive]: +runtime.gopark(0xc00106e5f0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e3def8 sp=0xc001e3ded8 pc=0x4777ae +runtime.chanrecv(0xc001143560, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001e3df70 sp=0xc001e3def8 pc=0x441325 +runtime.chanrecv1(0xc00106e5f0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001e3df98 sp=0xc001e3df70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc001e3dfe0 sp=0xc001e3df98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e3dfe8 sp=0xc001e3dfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 904 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 971 [chan receive]: +runtime.gopark(0xc000b3d0b0?, 0x28?, 0x0?, 0x0?, 0x460fed5?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d4aef8 sp=0xc001d4aed8 pc=0x4777ae +runtime.chanrecv(0xc001143560, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001d4af70 sp=0xc001d4aef8 pc=0x441325 +runtime.chanrecv1(0xc00106e5f0?, 0x47e975?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001d4af98 sp=0xc001d4af70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc001d4afe0 sp=0xc001d4af98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d4afe8 sp=0xc001d4afe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 969 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 864 [select]: +runtime.gopark(0xc001e3bf28?, 0x2?, 0xb4?, 0x1b?, 0xc001e3bebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e3bd30 sp=0xc001e3bd10 pc=0x4777ae +runtime.selectgo(0xc001e3bf28, 0xc001e3beb8, 0x0?, 0x0, 0xc001e3bf20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e3be80 sp=0xc001e3bd30 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc0019255f0, 0xc00149e420, 0xc0018f0c40) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001e3bf58 sp=0xc001e3be80 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeManagementAPI(0xc0019255f0, {0x5b6f770?, 0xc0010c0e70}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:185 +0x10a fp=0xc001e3bf98 sp=0xc001e3bf58 pc=0x21a7e0a +github.com/rancher/opni/pkg/management.NewServer.func1.2() + /home/ec2-user/opni/pkg/management/server.go:156 +0x62 fp=0xc001e3bfe0 sp=0xc001e3bf98 pc=0x231c0c2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e3bfe8 sp=0xc001e3bfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.NewServer.func1 in goroutine 860 + /home/ec2-user/opni/pkg/management/server.go:156 +0x137 + +goroutine 907 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002063978 sp=0xc002063958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0020639b0 sp=0xc002063978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163b948, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0020639d0 sp=0xc0020639b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0017cfca0, 0xc002063a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002063a18 sp=0xc0020639d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0017cfc80) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc002063b08 sp=0xc002063a18 pc=0x513f65 +net.(*netFD).accept(0xc0017cfc80) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc002063c00 sp=0xc002063b08 pc=0x8967be +net.(*UnixListener).accept(0xc001be2480) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc002063c40 sp=0xc002063c00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc001be2480) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc002063cb0 sp=0xc002063c40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc00127d4c0) + :1 +0x44 fp=0xc002063ce8 sp=0xc002063cb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc001ebfe00, {0x5b2c6a0, 0xc00127d4c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc002063f18 sp=0xc002063ce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc002063f68 sp=0xc002063f18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc00127d880?, 0xc0018f11a0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc002063fa8 sp=0xc002063f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc002063fe0 sp=0xc002063fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002063fe8 sp=0xc002063fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 904 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 762 [select]: +runtime.gopark(0xc001b6af20?, 0x2?, 0xb4?, 0x1b?, 0xc001b6aeb4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b6ad28 sp=0xc001b6ad08 pc=0x4777ae +runtime.selectgo(0xc001b6af20, 0xc001b6aeb0, 0xc001b6af98?, 0x0, 0xc001b6af18?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b6ae78 sp=0xc001b6ad28 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc001924a80, 0xc00114dcd0, 0xc00114dce0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001b6af50 sp=0xc001b6ae78 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeKeyValueStore(0xc001924a80, {0x5b43370?, 0xc00114dcc0}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:214 +0x1a8 fp=0xc001b6af98 sp=0xc001b6af50 pc=0x21a84a8 +github.com/rancher/opni/pkg/gateway.NewGateway.func2.1() + /home/ec2-user/opni/pkg/gateway/gateway.go:157 +0x62 fp=0xc001b6afe0 sp=0xc001b6af98 pc=0x22b18c2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b6afe8 sp=0xc001b6afe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.NewGateway.func2 in goroutine 814 + /home/ec2-user/opni/pkg/gateway/gateway.go:157 +0x3cc + +goroutine 880 [select]: +runtime.gopark(0xc0030c1e70?, 0x2?, 0x0?, 0x0?, 0xc0030c1df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0030c1c58 sp=0xc0030c1c38 pc=0x4777ae +runtime.selectgo(0xc0030c1e70, 0xc0030c1df0, 0xc002520690?, 0x0, 0xc0017cae00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0030c1da8 sp=0xc0030c1c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001abbe00, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0030c1ea0 sp=0xc0030c1da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002520690) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0030c1f58 sp=0xc0030c1ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc0030c1fe0 sp=0xc0030c1f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0030c1fe8 sp=0xc0030c1fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1045 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 844 [select]: +runtime.gopark(0xc001b68fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001b68f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b68de8 sp=0xc001b68dc8 pc=0x4777ae +runtime.selectgo(0xc001b68fb0, 0xc001b68f70, 0xc000081788?, 0x0, 0xc0010880c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b68f38 sp=0xc001b68de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001b68fe0 sp=0xc001b68f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b68fe8 sp=0xc001b68fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 845 [select]: +runtime.gopark(0xc001b67fb0?, 0x2?, 0xb0?, 0x7d?, 0xc001b67ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b67d58 sp=0xc001b67d38 pc=0x4777ae +runtime.selectgo(0xc001b67fb0, 0xc001b67ee0, 0x4aead9?, 0x0, 0x2106cfa?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b67ea8 sp=0xc001b67d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001b67fe0 sp=0xc001b67ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b67fe8 sp=0xc001b67fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 846 [chan receive]: +runtime.gopark(0xc001b66ce8?, 0x11775cf?, 0xd0?, 0x6e?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b66d60 sp=0xc001b66d40 pc=0x4777ae +runtime.chanrecv(0xc001088c60, 0xc001b66e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b66dd8 sp=0xc001b66d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b66e00 sp=0xc001b66dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001b66fe0 sp=0xc001b66e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b66fe8 sp=0xc001b66fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 764 [select]: +runtime.gopark(0xc00154d448?, 0x2?, 0xd0?, 0xd2?, 0xc00154d3fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00154d278 sp=0xc00154d258 pc=0x4777ae +runtime.selectgo(0xc00154d448, 0xc00154d3f8, 0xc0015113b8?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00154d3c8 sp=0xc00154d278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008a3200) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc00154d478 sp=0xc00154d3c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00184b110, {0x42219c0?, 0xc001c16420}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc00154d6d8 sp=0xc00154d478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0008a3074?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc00154d738 sp=0xc00154d6d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc0008a2fc0, 0xc00154d8f0, 0xc00154d8c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc00154d7e0 sp=0xc00154d738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0008a2fc0, {0x42219c0, 0xc001c16420}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc00154d920 sp=0xc00154d7e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc001c164e0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc001c163f0}, {0x42219c0, 0xc001c16420}, 0x0?, {0xc001f0f380, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc00154d9c0 sp=0xc00154d920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc001fefae0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc001c163f0}, {0x42219c0, 0xc001c16420}, 0xc001838400, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc00154dd30 sp=0xc00154d9c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc001838400, {0x5b426e0, 0xc001fefae0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc001c163f0}, {0x42219c0, 0xc001c16420}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc00154de58 sp=0xc00154dd30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseKeyValueStore(0xc00158bb70, {0x5b426e0, 0xc001fefae0}, 0xc001707750?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:69 +0xee fp=0xc00154dee8 sp=0xc00154de58 pc=0x21ab96e +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeKeyValueStore.func2(0x3) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:219 +0xf2 fp=0xc00154df58 sp=0xc00154dee8 pc=0x21a85d2 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc00154dfe0 sp=0xc00154df58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00154dfe8 sp=0xc00154dfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 762 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 831 [select]: +runtime.gopark(0xc001d79f70?, 0x2?, 0xb4?, 0x1b?, 0xc001d79ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d79d20 sp=0xc001d79d00 pc=0x4777ae +runtime.selectgo(0xc001d79f70, 0xc001d79eb8, 0x4aead9?, 0x0, 0x215774d?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001d79e70 sp=0xc001d79d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001d861e0, {0x5b426e0, 0xc001abad70}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001d79fa0 sp=0xc001d79e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001d79fe0 sp=0xc001d79fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d79fe8 sp=0xc001d79fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 824 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 832 [select]: +runtime.gopark(0xc001a05f70?, 0x2?, 0xb4?, 0x1b?, 0xc001a05ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a05d20 sp=0xc001a05d00 pc=0x4777ae +runtime.selectgo(0xc001a05f70, 0xc001a05eb8, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001a05e70 sp=0xc001a05d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001d86240, {0x5b426e0, 0xc001abadc0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001a05fa0 sp=0xc001a05e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001a05fe0 sp=0xc001a05fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a05fe8 sp=0xc001a05fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 824 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 899 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0xc0?, 0xd3?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d77cd0 sp=0xc001d77cb0 pc=0x4777ae +runtime.chanrecv(0xc001a3b860, 0xc001d77de0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001d77d48 sp=0xc001d77cd0 pc=0x441325 +runtime.chanrecv1(0xc001d77df0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001d77d70 sp=0xc001d77d48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc001d77f30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc001d77e00 sp=0xc001d77d70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc0011f24b0, 0x420c60?, 0xc001be2270) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc001d77fa0 sp=0xc001d77e00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc001d77fe0 sp=0xc001d77fa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d77fe8 sp=0xc001d77fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 864 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 847 [select]: +runtime.gopark(0xc001fbe890?, 0x2?, 0x98?, 0xe7?, 0xc001fbe83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fbe6a8 sp=0xc001fbe688 pc=0x4777ae +runtime.selectgo(0xc001fbe890, 0xc001fbe838, 0x7f429a45e200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001fbe7f8 sp=0xc001fbe6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00128db30, {0xc001b402e0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc001fbe8c0 sp=0xc001fbe7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00128db30, {0xc001b402e0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001fbe958 sp=0xc001fbe8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001925a10, {0xc001b402e0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001fbe9a0 sp=0xc001fbe958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001925a10}, {0xc001b402e0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001fbe9e8 sp=0xc001fbe9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001196900, {0xc001b402e0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001fbea58 sp=0xc001fbe9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001b402d0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001fbeb28 sp=0xc001fbea58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001b09e00?, 0xc001196900, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001fbec18 sp=0xc001fbeb28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001b402d0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001b6e1e0}, 0xc0010e4480?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001fbecd0 sp=0xc001fbec18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001b9c690, {0x43dcf60?, 0xc001b6e1e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001fbef08 sp=0xc001fbecd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001b40420, {0x43dcf60, 0xc001b6e1e0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001fbeff0 sp=0xc001fbef08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc000b30f00, {0x43dcf60, 0xc001b6e1e0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001fbf028 sp=0xc001fbeff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc0018be7b0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001fbf068 sp=0xc001fbf028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc0018be7b0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc001fbf0a8 sp=0xc001fbf068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc0018be7d0) + :1 +0x46 fp=0xc001fbf0e0 sp=0xc001fbf0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0010e4630, {0x5b51368, 0xc0018be7d0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001fbf320 sp=0xc001fbf0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00158b250, {0x5b51038, 0xc0018be7b0}) + :1 +0xaf fp=0xc001fbf370 sp=0xc001fbf320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00158b250}, {0x5b4a6f0?, 0xc000b30f00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc001fbf3b0 sp=0xc001fbf370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001b40420}, 0xc000126fc0?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001fbf400 sp=0xc001fbf3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001b40420}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001fbf490 sp=0xc001fbf400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001b9c690}, 0xc000126fc0, 0xc001b09400) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001fbf7d0 sp=0xc001fbf490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001b9c690}, 0x3f252a0?, 0xc0018be6e0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001fbf848 sp=0xc001fbf7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc001196900, 0xc0010e4810, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001fbfdb0 sp=0xc001fbf848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc001196900, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001fbff28 sp=0xc001fbfdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001fbffe0 sp=0xc001fbff28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fbffe8 sp=0xc001fbffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1155 [chan receive]: +runtime.gopark(0x3fa3c40?, 0xc002578d20?, 0x0?, 0x0?, 0xc00213df00?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00213df08 sp=0xc00213dee8 pc=0x4777ae +runtime.chanrecv(0xc002578d20, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00213df80 sp=0xc00213df08 pc=0x441325 +runtime.chanrecv1(0xc0024638b0?, 0xc001ebfc20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00213dfa8 sp=0xc00213df80 pc=0x440f72 +github.com/nats-io/nats%2ego.(*js).subscribe.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/js.go:1817 +0x4a fp=0xc00213dfe0 sp=0xc00213dfa8 pc=0x1f4624a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00213dfe8 sp=0xc00213dfe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*js).subscribe in goroutine 1133 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/js.go:1816 +0x2c85 + +goroutine 766 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e5b978 sp=0xc001e5b958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001e5b9b0 sp=0xc001e5b978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163c010, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001e5b9d0 sp=0xc001e5b9b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00182da20, 0xc001e5ba01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001e5ba18 sp=0xc001e5b9d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc00182da00) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001e5bb08 sp=0xc001e5ba18 pc=0x513f65 +net.(*netFD).accept(0xc00182da00) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001e5bc00 sp=0xc001e5bb08 pc=0x8967be +net.(*UnixListener).accept(0xc001c166c0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc001e5bc40 sp=0xc001e5bc00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc001c166c0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc001e5bcb0 sp=0xc001e5bc40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc001d4c120) + :1 +0x44 fp=0xc001e5bce8 sp=0xc001e5bcb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc0018525a0, {0x5b2c6a0, 0xc001d4c120}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc001e5bf18 sp=0xc001e5bce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc001e5bf68 sp=0xc001e5bf18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc001d4c280?, 0xc00114df70?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001e5bfa8 sp=0xc001e5bf68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001e5bfe0 sp=0xc001e5bfa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e5bfe8 sp=0xc001e5bfe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 763 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 767 [select]: +runtime.gopark(0xc001b6bf38?, 0x2?, 0x0?, 0x0?, 0xc001b6bf14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b6bd90 sp=0xc001b6bd70 pc=0x4777ae +runtime.selectgo(0xc001b6bf38, 0xc001b6bf10, 0xc001b6bf18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b6bee0 sp=0xc001b6bd90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc001b6bf68 sp=0xc001b6bee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc000dc2078?, 0xc00114df90?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001b6bfa8 sp=0xc001b6bf68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001b6bfe0 sp=0xc001b6bfa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b6bfe8 sp=0xc001b6bfe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 763 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 871 [IO wait]: +runtime.gopark(0xc001fc4fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fc4f98 sp=0xc001fc4f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001fc4fd0 sp=0xc001fc4f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163bf18, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001fc4ff0 sp=0xc001fc4fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001ab06a0, 0xc0025ce000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001fc5038 sp=0xc001fc4ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001ab0680, {0xc0025ce000, 0x4000, 0x4000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001fc5128 sp=0xc001fc5038 pc=0x50e5a5 +net.(*netFD).Read(0xc001ab0680, {0xc0025ce000, 0x4000, 0x4000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001fc5188 sp=0xc001fc5128 pc=0x89342b +net.(*conn).Read(0xc000b79f20, {0xc0025ce000, 0x4000, 0x4000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001fc5210 sp=0xc001fc5188 pc=0x8b176d +net.(*UnixConn).Read(0xc000b79f20, {0xc0025ce000, 0x4000, 0x4000}) + :1 +0x54 fp=0xc001fc5258 sp=0xc001fc5210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc000dc3770, {0xc0025ce000, 0x4000, 0x4000}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001fc52b8 sp=0xc001fc5258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc00189f7a8, {0x5b0f6c0, 0xc000dc3770}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc001fc5320 sp=0xc001fc52b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc00189f500, {0x5b0f840?, 0xc000b79f20}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001fc5390 sp=0xc001fc5320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc00189f500, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001fc59c0 sp=0xc001fc5390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc00189f500, {0xc002200000, 0x8000, 0xc002339720?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001fc5aa8 sp=0xc001fc59c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001d86780, {0xc002200000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001fc5af0 sp=0xc001fc5aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc001de6060, {0xc0011d59a0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001fc5bb0 sp=0xc001fc5af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001de6060}, {0xc0011d59a0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001fc5bf8 sp=0xc001fc5bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0011d59a0, 0x9, 0x9}, {0x5b0f0a0, 0xc001de6060}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001fc5c98 sp=0xc001fc5bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011d5960) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001fc5dd0 sp=0xc001fc5c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001df0480, 0x1157434?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc001fc5fb0 sp=0xc001fc5dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc001fc5fe0 sp=0xc001fc5fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fc5fe8 sp=0xc001fc5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 833 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 768 [chan receive]: +runtime.gopark(0xc001ded5e4?, 0x4be6c0?, 0xe0?, 0xd5?, 0x4be720?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001eeedd0 sp=0xc001eeedb0 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001eeee48 sp=0xc001eeedd0 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0xc00237c720?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001eeee70 sp=0xc001eeee48 pc=0x440f72 +github.com/rancher/opni/plugins/metrics/pkg/gateway.(*Plugin).UseKeyValueStore(0xc000379180, {0x5b43f90?, 0xc001ab6550}) + /home/ec2-user/opni/plugins/metrics/pkg/gateway/system.go:91 +0x610 fp=0xc001eeefd0 sp=0xc001eeee70 pc=0x3db4bf0 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseKeyValueStore(0xc001160340, {0x4b05a9?, 0xc001e5d0f0?}, 0xc001c16a80) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:123 +0x18d fp=0xc001eef098 sp=0xc001eeefd0 pc=0x21a6c2d +github.com/rancher/opni/pkg/plugins/apis/system._System_UseKeyValueStore_Handler.func1({0x5b426a8, 0xc001c16b40}, {0x4278de0?, 0xc001c16a80}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:187 +0x83 fp=0xc001eef0f0 sp=0xc001eef098 pc=0x21ac823 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc001c16b40}, {0x4278de0, 0xc001c16a80}, 0xc001d4c360, 0xc000dc21e0) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc001eef148 sp=0xc001eef0f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc001c16b40}, {0x4278de0, 0xc001c16a80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc001eef1e8 sp=0xc001eef148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc001c16a50}, {0x4278de0, 0xc001c16a80}, 0xc001d4c360, 0xc0019f1a00) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc001eef6b8 sp=0xc001eef1e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc001c16a50}, {0x4278de0, 0xc001c16a80}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc001eef740 sp=0xc001eef6b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseKeyValueStore_Handler({0x4278ea0?, 0xc001160340}, {0x5b426a8, 0xc001c16a50}, 0xc002013260, 0xc001ded8a0) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:189 +0x1c4 fp=0xc001eef7d0 sp=0xc001eef740 pc=0x21ac684 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc0008a3320, 0xc0010e57d0, 0x771ecd0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc001eefdb0 sp=0xc001eef7d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc0008a3320, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc001eeff28 sp=0xc001eefdb0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001eeffe0 sp=0xc001eeff28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001eeffe8 sp=0xc001eeffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1015 [select]: +runtime.gopark(0xc001ef0f70?, 0x4?, 0xa9?, 0x5?, 0xc001ef0dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ef0c00 sp=0xc001ef0be0 pc=0x4777ae +runtime.selectgo(0xc001ef0f70, 0xc001ef0dd0, 0x9000000000000000?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ef0d50 sp=0xc001ef0c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002135a00) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc001ef0fc0 sp=0xc001ef0d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc001ef0fe0 sp=0xc001ef0fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ef0fe8 sp=0xc001ef0fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 966 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 970 [select]: +runtime.gopark(0xc00211b448?, 0x2?, 0xd0?, 0xb2?, 0xc00211b3fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00211b278 sp=0xc00211b258 pc=0x4777ae +runtime.selectgo(0xc00211b448, 0xc00211b3f8, 0xc00158b938?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00211b3c8 sp=0xc00211b278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001440b40) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc00211b478 sp=0xc00211b3c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001d51040, {0x42219c0?, 0xc002008d20}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc00211b6d8 sp=0xc00211b478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0014409b4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc00211b738 sp=0xc00211b6d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001440900, 0xc00211b8f0, 0xc00211b8c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc00211b7e0 sp=0xc00211b738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001440900, {0x42219c0, 0xc002008d20}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc00211b920 sp=0xc00211b7e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc002008de0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc002008cf0}, {0x42219c0, 0xc002008d20}, 0x0?, {0xc000ee7320, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc00211b9c0 sp=0xc00211b920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc00106e5f0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc002008cf0}, {0x42219c0, 0xc002008d20}, 0xc0018aa800, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc00211bd30 sp=0xc00211b9c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc0018aa800, {0x5b426e0, 0xc00106e5f0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc002008cf0}, {0x42219c0, 0xc002008d20}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc00211be58 sp=0xc00211bd30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseKeyValueStore(0xc0018be2e0, {0x5b426e0, 0xc00106e5f0}, 0x0?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:69 +0xee fp=0xc00211bee8 sp=0xc00211be58 pc=0x21ab96e +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeKeyValueStore.func2(0x3) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:219 +0xf2 fp=0xc00211bf58 sp=0xc00211bee8 pc=0x21a85d2 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc00211bfe0 sp=0xc00211bf58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00211bfe8 sp=0xc00211bfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 968 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 1190 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00210a830 sp=0xc00210a810 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00210a868 sp=0xc00210a830 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494528, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00210a888 sp=0xc00210a868 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001508420, 0xc001824000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00210a8d0 sp=0xc00210a888 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001508400, {0xc001824000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00210a9c0 sp=0xc00210a8d0 pc=0x50e5a5 +net.(*netFD).Read(0xc001508400, {0xc001824000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00210aa20 sp=0xc00210a9c0 pc=0x89342b +net.(*conn).Read(0xc001ce3b58, {0xc001824000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00210aaa8 sp=0xc00210aa20 pc=0x8b176d +net.(*TCPConn).Read(0xc001ce3b58, {0xc001824000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc00210aaf0 sp=0xc00210aaa8 pc=0x8d72f4 +bufio.(*Reader).Read(0xc0014ac1e0, {0xc0014f8660, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00210abb0 sp=0xc00210aaf0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0014ac1e0}, {0xc0014f8660, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00210abf8 sp=0xc00210abb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0014f8660, 0x9, 0x9}, {0x5b0f0a0, 0xc0014ac1e0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00210ac98 sp=0xc00210abf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0014f8620) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00210add0 sp=0xc00210ac98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000776240, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc00210afb0 sp=0xc00210add0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc00210afe0 sp=0xc00210afb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00210afe8 sp=0xc00210afe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1073 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 956 [chan receive]: +runtime.gopark(0x400a060?, 0xc001e2fcc0?, 0xe3?, 0x45?, 0x7?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029ddb28 sp=0xc0029ddb08 pc=0x4777ae +runtime.chanrecv(0xc002080e40, 0xc0020f3cb8, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0029ddba0 sp=0xc0029ddb28 pc=0x441325 +runtime.chanrecv1(0xc001ff69c0?, 0x771d460?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0029ddbc8 sp=0xc0029ddba0 pc=0x440f72 +github.com/rancher/opni/pkg/gateway.(*StreamServer).OnPluginLoad.func1() + /home/ec2-user/opni/pkg/gateway/stream.go:285 +0x8eb fp=0xc0029ddfe0 sp=0xc0029ddbc8 pc=0x22bfbeb +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029ddfe8 sp=0xc0029ddfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.(*StreamServer).OnPluginLoad in goroutine 934 + /home/ec2-user/opni/pkg/gateway/stream.go:255 +0xa52 + +goroutine 769 [select]: +runtime.gopark(0xc001fb5f70?, 0x2?, 0xb4?, 0x1b?, 0xc001fb5ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fb5d20 sp=0xc001fb5d00 pc=0x4777ae +runtime.selectgo(0xc001fb5f70, 0xc001fb5eb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001fb5e70 sp=0xc001fb5d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001d4c540, {0x5b426e0, 0xc001c64780}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001fb5fa0 sp=0xc001fb5e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001fb5fe0 sp=0xc001fb5fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fb5fe8 sp=0xc001fb5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 768 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 882 [select]: +runtime.gopark(0xc001e5df70?, 0x2?, 0xb4?, 0x1b?, 0xc001e5debc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e5dd20 sp=0xc001e5dd00 pc=0x4777ae +runtime.selectgo(0xc001e5df70, 0xc001e5deb8, 0x4b3745?, 0x0, 0xc001fefb48?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e5de70 sp=0xc001e5dd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001d4c5a0, {0x5b426e0, 0xc001c647d0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001e5dfa0 sp=0xc001e5de70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001e5dfe0 sp=0xc001e5dfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e5dfe8 sp=0xc001e5dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 768 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 865 [select]: +runtime.gopark(0xc001e3cf28?, 0x2?, 0xb4?, 0x1b?, 0xc001e3cebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e3cd30 sp=0xc001e3cd10 pc=0x4777ae +runtime.selectgo(0xc001e3cf28, 0xc001e3ceb8, 0x22c21ff?, 0x0, 0xc001e3cf20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e3ce80 sp=0xc001e3cd30 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc0019255f0, 0xc00149e510, 0xc0018f0e10) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001e3cf58 sp=0xc001e3ce80 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeNodeManagerServer(0xc0019255f0, {0x5b21b70?, 0xc0017e1740}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:198 +0x10a fp=0xc001e3cf98 sp=0xc001e3cf58 pc=0x21a810a +github.com/rancher/opni/pkg/management.NewServer.func1.3() + /home/ec2-user/opni/pkg/management/server.go:158 +0x62 fp=0xc001e3cfe0 sp=0xc001e3cf98 pc=0x231c022 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e3cfe8 sp=0xc001e3cfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.NewServer.func1 in goroutine 860 + /home/ec2-user/opni/pkg/management/server.go:158 +0x245 + +goroutine 1174 [select]: +runtime.gopark(0xc002c92828?, 0x2?, 0x8?, 0x0?, 0xc002c9252c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c92300 sp=0xc002c922e0 pc=0x4777ae +runtime.selectgo(0xc002c92828, 0xc002c92528, 0xc002c7f8d0?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c92450 sp=0xc002c92300 pc=0x48916b +github.com/rancher/opni/pkg/gateway.(*StreamServer).Connect(0xc0020917c0, {0x5b508a0, 0xc002857580}) + /home/ec2-user/opni/pkg/gateway/stream.go:181 +0x1e05 fp=0xc002c92e80 sp=0xc002c92450 pc=0x22bcf85 +github.com/rancher/opni/pkg/apis/stream/v1._Stream_Connect_Handler({0x432f2e0?, 0xc0020917c0}, {0x5b4ab30?, 0xc0027cc1e0}) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:121 +0xb6 fp=0xc002c92ec0 sp=0xc002c92e80 pc=0x2003ed6 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x432f2e0, 0xc0020917c0}, {0x5b4a6f0, 0xc002488960}, 0xc001e46060, 0x47ee1d0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002c93200 sp=0xc002c92ec0 pc=0x12b0e02 +google.golang.org/grpc.getChainStreamHandler.func1({0x432f2e0, 0xc0020917c0}, {0x5b4a6f0, 0xc002488960}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002c93290 sp=0xc002c93200 pc=0x117fe75 +github.com/rancher/opni/pkg/gateway.NewGateway.NewLastKnownDetailsApplier.func16({0x432f2e0, 0xc0020917c0}, {0x5b4a6f0?, 0xc002488960}, 0xc001e46060?, 0xc001d7d9c0) + /home/ec2-user/opni/pkg/gateway/versions.go:54 +0x5e8 fp=0xc002c93388 sp=0xc002c93290 pc=0x22af588 +google.golang.org/grpc.getChainStreamHandler.func1({0x432f2e0, 0xc0020917c0}, {0x5b4a6f0, 0xc002488960}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002c93418 sp=0xc002c93388 pc=0x117fe75 +github.com/rancher/opni/pkg/gateway.NewGateway.(*UpdateServer).StreamServerInterceptor.func15({0x432f2e0, 0xc0020917c0}, {0x5b4a6f0, 0xc002bf8da0}, 0xc001e46060?, 0xc001d7d880) + /home/ec2-user/opni/pkg/update/server.go:150 +0xbac fp=0xc002c93580 sp=0xc002c93418 pc=0x22b02ac +google.golang.org/grpc.getChainStreamHandler.func1({0x432f2e0, 0xc0020917c0}, {0x5b4a6f0, 0xc002bf8da0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002c93610 sp=0xc002c93580 pc=0x117fe75 +github.com/rancher/opni/pkg/gateway.NewGateway.StreamServerInterceptor.func9({0x432f2e0, 0xc0020917c0}, {0x5b492f8, 0xc001670d20}, 0xc001e46060?, 0xc00236da40) + /home/ec2-user/opni/pkg/auth/cluster/middleware.go:19 +0x199 fp=0xc002c93680 sp=0xc002c93610 pc=0x22b08f9 +google.golang.org/grpc.getChainStreamHandler.func1({0x432f2e0, 0xc0020917c0}, {0x5b492f8, 0xc001670d20}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002c93710 sp=0xc002c93680 pc=0x117fe75 +github.com/rancher/opni/pkg/gateway.NewGateway.(*RatelimiterInterceptor).StreamServerInterceptor.func14({0x432f2e0, 0xc0020917c0}, {0x5b492f8, 0xc001670d20}, 0xc001e46060, 0xc00236da00) + /home/ec2-user/opni/pkg/gateway/ratelimit.go:65 +0x7e fp=0xc002c93778 sp=0xc002c93710 pc=0x22b03be +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x432f2e0, 0xc0020917c0}, {0x5b492f8, 0xc001670d20}, 0x3f252a0?, 0xc0026f2720?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002c937f0 sp=0xc002c93778 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00170e000, {0x5b54e00, 0xc002515040}, 0xc001d8ad80, 0xc002172ae0, 0x770c800, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002c93d58 sp=0xc002c937f0 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00170e000, {0x5b54e00, 0xc002515040}, 0xc001d8ad80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002c93ed0 sp=0xc002c93d58 pc=0x1184745 +google.golang.org/grpc.(*Server).handleSingleStream(0x1?, 0xc002bf8d80) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:604 +0x165 fp=0xc002c93f88 sp=0xc002c93ed0 pc=0x1174725 +google.golang.org/grpc.(*Server).serverWorker(0xc00170e000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:597 +0x38 fp=0xc002c93fc0 sp=0xc002c93f88 pc=0x1174458 +google.golang.org/grpc.(*Server).initServerWorkers.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0x34 fp=0xc002c93fe0 sp=0xc002c93fc0 pc=0x1174994 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c93fe8 sp=0xc002c93fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).initServerWorkers in goroutine 1088 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0xa5 + +goroutine 1318 [chan receive]: +runtime.gopark(0xc0019f9e30?, 0x4c0446?, 0x0?, 0x0?, 0x4aead9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019f9e68 sp=0xc0019f9e48 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0019f9ee0 sp=0xc0019f9e68 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0019f9f08 sp=0xc0019f9ee0 pc=0x440f72 +github.com/rancher/opni/plugins/alerting/test.(*TestEnvAlertingClusterDriver).StartAlertingBackendServer.func1() + /home/ec2-user/opni/plugins/alerting/test/test_drivers.go:384 +0x5c fp=0xc0019f9f50 sp=0xc0019f9f08 pc=0x398719c +github.com/rancher/opni/pkg/util/waitctx.permissive.Go.func1() + /home/ec2-user/opni/pkg/util/waitctx/context.go:222 +0x93 fp=0xc0019f9fe0 sp=0xc0019f9f50 pc=0x178e3d3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0019f9fe8 sp=0xc0019f9fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/util/waitctx.permissive.Go in goroutine 1250 + /home/ec2-user/opni/pkg/util/waitctx/context.go:220 +0xeb + +goroutine 1081 [select]: +runtime.gopark(0xc001e64890?, 0x2?, 0xe0?, 0x46?, 0xc001e6483c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e646a8 sp=0xc001e64688 pc=0x4777ae +runtime.selectgo(0xc001e64890, 0xc001e64838, 0x7f429a45d300?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e647f8 sp=0xc001e646a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00255caa0, {0xc00253f5d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc001e648c0 sp=0xc001e647f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00255caa0, {0xc00253f5d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001e64958 sp=0xc001e648c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00253f530, {0xc00253f5d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001e649a0 sp=0xc001e64958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc00253f530}, {0xc00253f5d0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001e649e8 sp=0xc001e649a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000c04b40, {0xc00253f5d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001e64a58 sp=0xc001e649e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc00253f5c0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001e64b28 sp=0xc001e64a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc000fe6680?, 0xc000c04b40, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001e64c18 sp=0xc001e64b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc00253f5c0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001662f50}, 0x1?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001e64cd0 sp=0xc001e64c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001aae000, {0x43dcf60?, 0xc001662f50}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001e64f08 sp=0xc001e64cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc00253f710, {0x43dcf60, 0xc001662f50}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001e64ff0 sp=0xc001e64f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc001d87e60, {0x43dcf60, 0xc001662f50}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001e65028 sp=0xc001e64ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc002576540) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001e65068 sp=0xc001e65028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc002576540?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc001e650a8 sp=0xc001e65068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc002576550) + :1 +0x46 fp=0xc001e650e0 sp=0xc001e650a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc001266ea0, {0x5b51368, 0xc002576550}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001e65320 sp=0xc001e650e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc001510830, {0x5b51038, 0xc002576540}) + :1 +0xaf fp=0xc001e65370 sp=0xc001e65320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc001510830}, {0x5b4a6f0?, 0xc001d87e60}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc001e653b0 sp=0xc001e65370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc00253f710}, 0xc0012e5518?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001e65400 sp=0xc001e653b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc00253f710}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001e65490 sp=0xc001e65400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001aae000}, 0xc0012e5518, 0xc002570380) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001e657d0 sp=0xc001e65490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001aae000}, 0x3f252a0?, 0xc002576470?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001e65848 sp=0xc001e657d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc000c04b40, 0xc001266f30, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001e65db0 sp=0xc001e65848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc000c04b40, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001e65f28 sp=0xc001e65db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001e65fe0 sp=0xc001e65f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e65fe8 sp=0xc001e65fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 900 [select]: +runtime.gopark(0xc001b49448?, 0x2?, 0xd0?, 0x92?, 0xc001b493fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b49278 sp=0xc001b49258 pc=0x4777ae +runtime.selectgo(0xc001b49448, 0xc001b493f8, 0xc00158b938?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b493c8 sp=0xc001b49278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00123c240) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001b49478 sp=0xc001b493c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bd0c30, {0x42219c0?, 0xc001be24e0}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc001b496d8 sp=0xc001b49478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000926654?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001b49738 sp=0xc001b496d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc0009265a0, 0xc001b498f0, 0xc001b498c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc001b497e0 sp=0xc001b49738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0009265a0, {0x42219c0, 0xc001be24e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001b49920 sp=0xc001b497e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc001be25a0}, {0x4661da0, 0x1f}, {0x4278de0, 0xc001be24b0}, {0x42219c0, 0xc001be24e0}, 0x0?, {0xc000ee7320, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc001b499c0 sp=0xc001b49920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc00106e5f0}, {0x4661da0, 0x1f}, {0x4278de0, 0xc001be24b0}, {0x42219c0, 0xc001be24e0}, 0xc0018aa800, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc001b49d30 sp=0xc001b499c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc0018aa800, {0x5b426e0, 0xc00106e5f0}, {0x4661da0, 0x1f}, {0x4278de0, 0xc001be24b0}, {0x42219c0, 0xc001be24e0}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc001b49e58 sp=0xc001b49d30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseManagementAPI(0xc0018be2e0, {0x5b426e0, 0xc00106e5f0}, 0xc000086750?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:51 +0xee fp=0xc001b49ee8 sp=0xc001b49e58 pc=0x21ab66e +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeManagementAPI.func2(0x1) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:190 +0xf2 fp=0xc001b49f58 sp=0xc001b49ee8 pc=0x21a7f32 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc001b49fe0 sp=0xc001b49f58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b49fe8 sp=0xc001b49fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 864 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 1005 [select]: +runtime.gopark(0xc001633e48?, 0x2?, 0x0?, 0x0?, 0xc001633dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001633c30 sp=0xc001633c10 pc=0x4777ae +runtime.selectgo(0xc001633e48, 0xc001633dc8, 0xc001633dd8?, 0x0, 0xc00189fc00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001633d80 sp=0xc001633c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0021e8640, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001633e78 sp=0xc001633d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00243d810) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001633f30 sp=0xc001633e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc001633fe0 sp=0xc001633f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001633fe8 sp=0xc001633fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 873 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 969 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0xd5?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020efcd0 sp=0xc0020efcb0 pc=0x4777ae +runtime.chanrecv(0xc0020477a0, 0xc0020efde0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0020efd48 sp=0xc0020efcd0 pc=0x441325 +runtime.chanrecv1(0xc0020efdf0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0020efd70 sp=0xc0020efd48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc0020eff30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc0020efe00 sp=0xc0020efd70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc0011f24b0, 0x420c60?, 0xc002008c90) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc0020effa0 sp=0xc0020efe00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc0020effe0 sp=0xc0020effa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020effe8 sp=0xc0020effe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 968 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 901 [chan receive]: +runtime.gopark(0x22c2125?, 0xc000086f50?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d48ef8 sp=0xc001d48ed8 pc=0x4777ae +runtime.chanrecv(0xc001143560, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001d48f70 sp=0xc001d48ef8 pc=0x441325 +runtime.chanrecv1(0xc00106e5f0?, 0x22c2050?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001d48f98 sp=0xc001d48f70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc001d48fe0 sp=0xc001d48f98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d48fe8 sp=0xc001d48fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 899 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 902 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fc1978 sp=0xc001fc1958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001fc19b0 sp=0xc001fc1978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163ba40, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001fc19d0 sp=0xc001fc19b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0017cfb20, 0xc001fc1a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001fc1a18 sp=0xc001fc19d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0017cfb00) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001fc1b08 sp=0xc001fc1a18 pc=0x513f65 +net.(*netFD).accept(0xc0017cfb00) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001fc1c00 sp=0xc001fc1b08 pc=0x8967be +net.(*UnixListener).accept(0xc001be22a0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc001fc1c40 sp=0xc001fc1c00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc001be22a0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc001fc1cb0 sp=0xc001fc1c40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc00127d1c0) + :1 +0x44 fp=0xc001fc1ce8 sp=0xc001fc1cb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc001ebfc20, {0x5b2c6a0, 0xc00127d1c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc001fc1f18 sp=0xc001fc1ce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc001fc1f68 sp=0xc001fc1f18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc00127d340?, 0xc0018f0dc0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001fc1fa8 sp=0xc001fc1f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001fc1fe0 sp=0xc001fc1fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fc1fe8 sp=0xc001fc1fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 899 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 903 [select]: +runtime.gopark(0xc001982f38?, 0x2?, 0x69?, 0x97?, 0xc001982f14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001982d90 sp=0xc001982d70 pc=0x4777ae +runtime.selectgo(0xc001982f38, 0xc001982f10, 0xc001982f18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001982ee0 sp=0xc001982d90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc001982f68 sp=0xc001982ee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc00149e4f8?, 0xc0018f0de0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001982fa8 sp=0xc001982f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001982fe0 sp=0xc001982fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001982fe8 sp=0xc001982fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 899 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 904 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x40?, 0xd9?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fb3cd0 sp=0xc001fb3cb0 pc=0x4777ae +runtime.chanrecv(0xc001a3be60, 0xc001fb3de0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001fb3d48 sp=0xc001fb3cd0 pc=0x441325 +runtime.chanrecv1(0xc001fb3df0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001fb3d70 sp=0xc001fb3d48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc001fb3f30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc001fb3e00 sp=0xc001fb3d70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc0011f24b0, 0x2?, 0xc001be2450) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc001fb3fa0 sp=0xc001fb3e00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc001fb3fe0 sp=0xc001fb3fa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fb3fe8 sp=0xc001fb3fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 865 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 905 [select]: +runtime.gopark(0xc002031448?, 0x2?, 0xd0?, 0x12?, 0xc0020313fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002031278 sp=0xc002031258 pc=0x4777ae +runtime.selectgo(0xc002031448, 0xc0020313f8, 0xc00158b938?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020313c8 sp=0xc002031278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00123c6c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc002031478 sp=0xc0020313c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bd0d00, {0x42219c0?, 0xc001be27e0}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0020316d8 sp=0xc002031478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00123c414?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc002031738 sp=0xc0020316d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00123c360, 0xc0020318f0, 0xc0020318c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc0020317e0 sp=0xc002031738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00123c360, {0x42219c0, 0xc001be27e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc002031920 sp=0xc0020317e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc001be28a0}, {0x4673e80, 0x23}, {0x4278de0, 0xc001be27b0}, {0x42219c0, 0xc001be27e0}, 0x0?, {0xc000ee7320, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc0020319c0 sp=0xc002031920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc00106e5f0}, {0x4673e80, 0x23}, {0x4278de0, 0xc001be27b0}, {0x42219c0, 0xc001be27e0}, 0xc0018aa800, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc002031d30 sp=0xc0020319c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc0018aa800, {0x5b426e0, 0xc00106e5f0}, {0x4673e80, 0x23}, {0x4278de0, 0xc001be27b0}, {0x42219c0, 0xc001be27e0}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc002031e58 sp=0xc002031d30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseNodeManagerClient(0xc0018be2e0, {0x5b426e0, 0xc00106e5f0}, 0x0?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:60 +0xee fp=0xc002031ee8 sp=0xc002031e58 pc=0x21ab7ee +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeNodeManagerServer.func2(0x2) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:203 +0xf2 fp=0xc002031f58 sp=0xc002031ee8 pc=0x21a8232 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc002031fe0 sp=0xc002031f58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002031fe8 sp=0xc002031fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 865 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 1003 [select]: +runtime.gopark(0xc001587e70?, 0x2?, 0x0?, 0x0?, 0xc001587df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001587c58 sp=0xc001587c38 pc=0x4777ae +runtime.selectgo(0xc001587e70, 0xc001587df0, 0xc001587e00?, 0x0, 0xc001969180?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001587da8 sp=0xc001587c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0021e8230, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001587ea0 sp=0xc001587da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001b3ff10) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001587f58 sp=0xc001587ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc001587fe0 sp=0xc001587f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001587fe8 sp=0xc001587fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 927 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 914 [select]: +runtime.gopark(0xc0020f6890?, 0x2?, 0x98?, 0x67?, 0xc0020f683c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020f66a8 sp=0xc0020f6688 pc=0x4777ae +runtime.selectgo(0xc0020f6890, 0xc0020f6838, 0x7f429a458200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020f67f8 sp=0xc0020f66a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc001b6e370, {0xc001c17960, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0020f68c0 sp=0xc0020f67f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001b6e370, {0xc001c17960, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0020f6958 sp=0xc0020f68c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001b40720, {0xc001c17960, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0020f69a0 sp=0xc0020f6958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001b40720}, {0xc001c17960, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0020f69e8 sp=0xc0020f69a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001197440, {0xc001c17960, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0020f6a58 sp=0xc0020f69e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001c17950, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0020f6b28 sp=0xc0020f6a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001d7ce80?, 0xc001197440, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0020f6c18 sp=0xc0020f6b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001c17950, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001c65540}, 0xc001266cc0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0020f6cd0 sp=0xc0020f6c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001344d20, {0x43dcf60?, 0xc001c65540}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0020f6f08 sp=0xc0020f6cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001c17aa0, {0x43dcf60, 0xc001c65540}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0020f6ff0 sp=0xc0020f6f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc001d4cbe0, {0x43dcf60, 0xc001c65540}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0020f7028 sp=0xc0020f6ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc001ab6cf0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc0020f7068 sp=0xc0020f7028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc001ab6cf0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0020f70a8 sp=0xc0020f7068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc001ab6d00) + :1 +0x46 fp=0xc0020f70e0 sp=0xc0020f70a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc001266ea0, {0x5b51368, 0xc001ab6d00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc0020f7320 sp=0xc0020f70e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc001510830, {0x5b51038, 0xc001ab6cf0}) + :1 +0xaf fp=0xc0020f7370 sp=0xc0020f7320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc001510830}, {0x5b4a6f0?, 0xc001d4cbe0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0020f73b0 sp=0xc0020f7370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001c17aa0}, 0xc000dc2588?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0020f7400 sp=0xc0020f73b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001c17aa0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0020f7490 sp=0xc0020f7400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001344d20}, 0xc000dc2588, 0xc001d7c280) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0020f77d0 sp=0xc0020f7490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001344d20}, 0x3f252a0?, 0xc001ab6c20?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0020f7848 sp=0xc0020f77d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001197440, 0xc001266f30, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0020f7db0 sp=0xc0020f7848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001197440, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0020f7f28 sp=0xc0020f7db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0020f7fe0 sp=0xc0020f7f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020f7fe8 sp=0xc0020f7fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1388 [select]: +runtime.gopark(0xc002161fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002161f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002161de8 sp=0xc002161dc8 pc=0x4777ae +runtime.selectgo(0xc002161fb0, 0xc002161f70, 0xc002161fd0?, 0x0, 0xc002714a80?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002161f38 sp=0xc002161de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002161fe0 sp=0xc002161f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002161fe8 sp=0xc002161fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1361 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1228 [select]: +runtime.gopark(0xc00151df10?, 0x3?, 0xb4?, 0x1b?, 0xc00151de92?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00151dca8 sp=0xc00151dc88 pc=0x4777ae +runtime.selectgo(0xc00151df10, 0xc00151de8c, 0x5b126c0?, 0x0, 0x1?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00151ddf8 sp=0xc00151dca8 pc=0x48916b +google.golang.org/grpc.(*addrConn).resetTransport(0xc001f12f00) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/clientconn.go:1368 +0x3ef fp=0xc00151df50 sp=0xc00151ddf8 pc=0x116334f +google.golang.org/grpc.(*addrConn).connect(0xc001f12f00) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/clientconn.go:993 +0x23b fp=0xc00151dfc0 sp=0xc00151df50 pc=0x116033b +google.golang.org/grpc.(*acBalancerWrapper).Connect.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/balancer_conn_wrappers.go:398 +0x34 fp=0xc00151dfe0 sp=0xc00151dfc0 pc=0x1157434 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00151dfe8 sp=0xc00151dfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*acBalancerWrapper).Connect in goroutine 1037 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/balancer_conn_wrappers.go:398 +0xa7 + +goroutine 908 [select]: +runtime.gopark(0xc001e3ef38?, 0x2?, 0x0?, 0x0?, 0xc001e3ef14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e3ed90 sp=0xc001e3ed70 pc=0x4777ae +runtime.selectgo(0xc001e3ef38, 0xc001e3ef10, 0xc001e3ef18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e3eee0 sp=0xc001e3ed90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc001e3ef68 sp=0xc001e3eee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc00149e780?, 0xc0018f11c0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001e3efa8 sp=0xc001e3ef68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001e3efe0 sp=0xc001e3efa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e3efe8 sp=0xc001e3efe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 904 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 1320 [select]: +runtime.gopark(0xc002c30480?, 0x3?, 0xa0?, 0xea?, 0xc002bfae3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bef1d0 sp=0xc001bef1b0 pc=0x4777ae +runtime.selectgo(0xc002c30480, 0xc002bfae34, 0x0?, 0x0, 0xc0026f2460?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001bef320 sp=0xc001bef1d0 pc=0x48916b +reflect.rselect({0xc001bef508, 0x3, 0x7f4241693c48?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:589 +0x345 fp=0xc001bef3d0 sp=0xc001bef320 pc=0x4a8f05 +reflect.Select({0xc002686700, 0x3, 0x2?}) + /home/ec2-user/sdk/go1.21.0/src/reflect/value.go:3104 +0xb8a fp=0xc001bef598 sp=0xc001bef3d0 pc=0x57c44a +github.com/rancher/opni/pkg/util.WaitAll({0x5b426e0, 0xc002c2e550}, 0xc0026f2320, {0xc001bef990, 0x2, 0x2}) + /home/ec2-user/opni/pkg/util/servers.go:51 +0x3ea fp=0xc001bef718 sp=0xc001bef598 pc=0x176ae8a +github.com/rancher/opni/pkg/agent/v2.(*Agent).ListenAndServe(0xc0025ee340, {0x5b426e0, 0xc001fee820}) + /home/ec2-user/opni/pkg/agent/v2/agent.go:449 +0x12d3 fp=0xc001befd10 sp=0xc001bef718 pc=0x21bea33 +github.com/rancher/opni/pkg/test.(*Environment).StartAgent.func3() + /home/ec2-user/opni/pkg/test/environment.go:1976 +0x6ea fp=0xc001beff50 sp=0xc001befd10 pc=0x24c8b6a +github.com/rancher/opni/pkg/util/waitctx.permissive.Go.func1() + /home/ec2-user/opni/pkg/util/waitctx/context.go:222 +0x93 fp=0xc001beffe0 sp=0xc001beff50 pc=0x178e3d3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001beffe8 sp=0xc001beffe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/util/waitctx.permissive.Go in goroutine 609 + /home/ec2-user/opni/pkg/util/waitctx/context.go:220 +0xeb + +goroutine 998 [select]: +runtime.gopark(0xc0022f2890?, 0x2?, 0x98?, 0x27?, 0xc0022f283c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0022f26a8 sp=0xc0022f2688 pc=0x4777ae +runtime.selectgo(0xc0022f2890, 0xc0022f2838, 0x7f429a45ca00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0022f27f8 sp=0xc0022f26a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc001b6f6d0, {0xc0021bad30, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0022f28c0 sp=0xc0022f27f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001b6f6d0, {0xc0021bad30, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0022f2958 sp=0xc0022f28c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0021ba0c0, {0xc0021bad30, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0022f29a0 sp=0xc0022f2958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0021ba0c0}, {0xc0021bad30, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0022f29e8 sp=0xc0022f29a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00146aa20, {0xc0021bad30, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0022f2a58 sp=0xc0022f29e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0021bad20, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0022f2b28 sp=0xc0022f2a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0021e6380?, 0xc00146aa20, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0022f2c18 sp=0xc0022f2b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0021bad20, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0021e80a0}, 0xc0010e4480?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0022f2cd0 sp=0xc0022f2c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001b9ca50, {0x43dcf60?, 0xc0021e80a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0022f2f08 sp=0xc0022f2cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0021bae70, {0x43dcf60, 0xc0021e80a0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0022f2ff0 sp=0xc0022f2f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0021805a0, {0x43dcf60, 0xc0021e80a0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0022f3028 sp=0xc0022f2ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc001d2c390) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc0022f3068 sp=0xc0022f3028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc001d2c390?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0022f30a8 sp=0xc0022f3068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc001d2c3a0) + :1 +0x46 fp=0xc0022f30e0 sp=0xc0022f30a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0010e4630, {0x5b51368, 0xc001d2c3a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc0022f3320 sp=0xc0022f30e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00158b250, {0x5b51038, 0xc001d2c390}) + :1 +0xaf fp=0xc0022f3370 sp=0xc0022f3320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00158b250}, {0x5b4a6f0?, 0xc0021805a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0022f33b0 sp=0xc0022f3370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc0021bae70}, 0xc001e46a38?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0022f3400 sp=0xc0022f33b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc0021bae70}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0022f3490 sp=0xc0022f3400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001b9ca50}, 0xc001e46a38, 0xc001e41980) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0022f37d0 sp=0xc0022f3490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001b9ca50}, 0x3f252a0?, 0xc001d2c2c0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0022f3848 sp=0xc0022f37d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc00146aa20, 0xc0010e4810, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0022f3db0 sp=0xc0022f3848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc00146aa20, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0022f3f28 sp=0xc0022f3db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0022f3fe0 sp=0xc0022f3f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0022f3fe8 sp=0xc0022f3fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 886 [chan receive]: +runtime.gopark(0x4b05a9?, 0xc001f36e68?, 0x6d?, 0x2d?, 0xc000a72d68?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f36dd8 sp=0xc001f36db8 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001f36e50 sp=0xc001f36dd8 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0x5b6f168?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001f36e78 sp=0xc001f36e50 pc=0x440f72 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI(0xc00070a000, {0x5b6f168, 0xc001a7f950}) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:92 +0x619 fp=0xc001f36fa0 sp=0xc001f36e78 pc=0x396f019 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseManagementAPI(0xc001367cc0, {0x4b05a9?, 0xc0020650f0?}, 0xc001c174a0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:102 +0x30f fp=0xc001f37098 sp=0xc001f36fa0 pc=0x21a640f +github.com/rancher/opni/pkg/plugins/apis/system._System_UseManagementAPI_Handler.func1({0x5b426a8, 0xc001c17560}, {0x4278de0?, 0xc001c174a0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:151 +0x83 fp=0xc001f370f0 sp=0xc001f37098 pc=0x21ac063 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc001c17560}, {0x4278de0, 0xc001c174a0}, 0xc001d4c860, 0xc000dc24b0) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc001f37148 sp=0xc001f370f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc001c17560}, {0x4278de0, 0xc001c174a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc001f371e8 sp=0xc001f37148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc001c17470}, {0x4278de0, 0xc001c174a0}, 0xc001d4c860, 0xc0019f1e80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc001f376b8 sp=0xc001f371e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc001c17470}, {0x4278de0, 0xc001c174a0}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc001f37740 sp=0xc001f376b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseManagementAPI_Handler({0x4278ea0?, 0xc001367cc0}, {0x5b426a8, 0xc001c17470}, 0xc0020139d0, 0xc000ee6aa0) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:153 +0x1c4 fp=0xc001f377d0 sp=0xc001f37740 pc=0x21abec4 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001440000, 0xc001267440, 0x771eca0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc001f37db0 sp=0xc001f377d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001440000, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc001f37f28 sp=0xc001f37db0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001f37fe0 sp=0xc001f37f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f37fe8 sp=0xc001f37fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1016 [IO wait]: +runtime.gopark(0xc002232f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002232f40 sp=0xc002232f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002232f78 sp=0xc002232f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494bf0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002232f98 sp=0xc002232f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001f003a0, 0xc002026000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002232fe0 sp=0xc002232f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001f00380, {0xc002026000, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0022330d0 sp=0xc002232fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc001f00380, {0xc002026000, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc002233130 sp=0xc0022330d0 pc=0x89342b +net.(*conn).Read(0xc001ce2b28, {0xc002026000, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0022331b8 sp=0xc002233130 pc=0x8b176d +net.(*UnixConn).Read(0xc001ce2b28, {0xc002026000, 0x240, 0x240}) + :1 +0x54 fp=0xc002233200 sp=0xc0022331b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc000dc2fc0, {0xc002026000, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc002233260 sp=0xc002233200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0015210a8, {0x5b0f6c0, 0xc000dc2fc0}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0022332c8 sp=0xc002233260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc001520e00, {0x5b0f840?, 0xc001ce2b28}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc002233338 sp=0xc0022332c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc001520e00, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc002233968 sp=0xc002233338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc001520e00, {0xc00230a000, 0x8000, 0x0?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc002233a50 sp=0xc002233968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001fab3e0, {0xc00230a000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc002233a98 sp=0xc002233a50 pc=0x10969ed +bufio.(*Reader).Read(0xc002308360, {0xc000881b60, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc002233b58 sp=0xc002233a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc002308360}, {0xc000881b60, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002233ba0 sp=0xc002233b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000881b60, 0x9, 0x9}, {0x5b0f0a0, 0xc002308360}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc002233c40 sp=0xc002233ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000881b20) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc002233d78 sp=0xc002233c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002135a00, 0x0?, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc002233ec8 sp=0xc002233d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc001ebfe00, {0x5b54e00?, 0xc002135a00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc002233f80 sp=0xc002233ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc002233fe0 sp=0xc002233f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002233fe8 sp=0xc002233fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 966 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 887 [chan receive]: +runtime.gopark(0x4be6c0?, 0xc001266090?, 0x20?, 0xe7?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002112f08 sp=0xc002112ee8 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002112f80 sp=0xc002112f08 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0x5b11a20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002112fa8 sp=0xc002112f80 pc=0x440f72 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseNodeManagerClient(0xc00070a000, {0x5b11a20, 0xc001a7f4a0}) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:222 +0x93 fp=0xc002112fd0 sp=0xc002112fa8 pc=0x3972293 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseNodeManagerClient(0xc001367cc0, {0x4b05a9?, 0xc00205f0f0?}, 0xc001c175f0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:113 +0x18d fp=0xc002113098 sp=0xc002112fd0 pc=0x21a68cd +github.com/rancher/opni/pkg/plugins/apis/system._System_UseNodeManagerClient_Handler.func1({0x5b426a8, 0xc001c176b0}, {0x4278de0?, 0xc001c175f0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:169 +0x83 fp=0xc0021130f0 sp=0xc002113098 pc=0x21ac443 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc001c176b0}, {0x4278de0, 0xc001c175f0}, 0xc001d4c8e0, 0xc000dc24f8) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc002113148 sp=0xc0021130f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc001c176b0}, {0x4278de0, 0xc001c175f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc0021131e8 sp=0xc002113148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc001c175c0}, {0x4278de0, 0xc001c175f0}, 0xc001d4c8e0, 0xc0019f1f80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc0021136b8 sp=0xc0021131e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc001c175c0}, {0x4278de0, 0xc001c175f0}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc002113740 sp=0xc0021136b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseNodeManagerClient_Handler({0x4278ea0?, 0xc001367cc0}, {0x5b426a8, 0xc001c175c0}, 0xc002013b90, 0xc000ee6aa0) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:171 +0x1c4 fp=0xc0021137d0 sp=0xc002113740 pc=0x21ac2a4 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001440120, 0xc001267440, 0x771ecb8, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc002113db0 sp=0xc0021137d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001440120, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc002113f28 sp=0xc002113db0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002113fe0 sp=0xc002113f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002113fe8 sp=0xc002113fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 931 [select]: +runtime.gopark(0xc0020c5fb0?, 0x2?, 0xb4?, 0x1b?, 0xc0020c5f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020c5de8 sp=0xc0020c5dc8 pc=0x4777ae +runtime.selectgo(0xc0020c5fb0, 0xc0020c5f70, 0x4b05a9?, 0x0, 0x8d0f55?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020c5f38 sp=0xc0020c5de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0020c5fe0 sp=0xc0020c5f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020c5fe8 sp=0xc0020c5fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 888 [select]: +runtime.gopark(0xc0020edf70?, 0x2?, 0xb4?, 0x1b?, 0xc0020edebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020edd20 sp=0xc0020edd00 pc=0x4777ae +runtime.selectgo(0xc0020edf70, 0xc0020edeb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020ede70 sp=0xc0020edd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001d4caa0, {0x5b426e0, 0xc001c65180}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0020edfa0 sp=0xc0020ede70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0020edfe0 sp=0xc0020edfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020edfe8 sp=0xc0020edfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 915 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 889 [select]: +runtime.gopark(0xc002061f70?, 0x2?, 0xb4?, 0x1b?, 0xc002061ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002061d20 sp=0xc002061d00 pc=0x4777ae +runtime.selectgo(0xc002061f70, 0xc002061eb8, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002061e70 sp=0xc002061d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001d4cb00, {0x5b426e0, 0xc001c651d0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc002061fa0 sp=0xc002061e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc002061fe0 sp=0xc002061fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002061fe8 sp=0xc002061fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 915 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 932 [select]: +runtime.gopark(0xc001ef2fb0?, 0x2?, 0xb0?, 0x2d?, 0xc001ef2ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ef2d58 sp=0xc001ef2d38 pc=0x4777ae +runtime.selectgo(0xc001ef2fb0, 0xc001ef2ee0, 0x8ab66c?, 0x0, 0x4b0571?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ef2ea8 sp=0xc001ef2d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001ef2fe0 sp=0xc001ef2ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ef2fe8 sp=0xc001ef2fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 933 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ef1d60 sp=0xc001ef1d40 pc=0x4777ae +runtime.chanrecv(0xc001be4c60, 0xc001ef1e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001ef1dd8 sp=0xc001ef1d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001ef1e00 sp=0xc001ef1dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001ef1fe0 sp=0xc001ef1e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ef1fe8 sp=0xc001ef1fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 957 [select]: +runtime.gopark(0xc00210efb0?, 0x2?, 0x0?, 0xee?, 0xc00210ef2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00210eda8 sp=0xc00210ed88 pc=0x4777ae +runtime.selectgo(0xc00210efb0, 0xc00210ef28, 0x0?, 0x0, 0x13?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00210eef8 sp=0xc00210eda8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc00210efe0 sp=0xc00210eef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00210efe8 sp=0xc00210efe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 956 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 935 [select]: +runtime.gopark(0xc0020c1fb0?, 0x2?, 0xb4?, 0x1b?, 0xc0020c1f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020c1de8 sp=0xc0020c1dc8 pc=0x4777ae +runtime.selectgo(0xc0020c1fb0, 0xc0020c1f70, 0xc001bfe2a0?, 0x0, 0xc001be4cc0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020c1f38 sp=0xc0020c1de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0020c1fe0 sp=0xc0020c1f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020c1fe8 sp=0xc0020c1fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 934 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 936 [select]: +runtime.gopark(0xc0020c2fb0?, 0x2?, 0x10?, 0xf1?, 0xc0020c2ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020c2d58 sp=0xc0020c2d38 pc=0x4777ae +runtime.selectgo(0xc0020c2fb0, 0xc0020c2ee0, 0x1?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020c2ea8 sp=0xc0020c2d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0020c2fe0 sp=0xc0020c2ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020c2fe8 sp=0xc0020c2fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 934 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 937 [chan receive]: +runtime.gopark(0xc00036b950?, 0xc00036bc20?, 0xb0?, 0xbc?, 0xc00043a520?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020c3d60 sp=0xc0020c3d40 pc=0x4777ae +runtime.chanrecv(0xc001be4f60, 0xc0020c3e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0020c3dd8 sp=0xc0020c3d60 pc=0x441325 +runtime.chanrecv1(0xc00036afc0?, 0xc00036b710?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0020c3e00 sp=0xc0020c3dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc0020c3fe0 sp=0xc0020c3e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020c3fe8 sp=0xc0020c3fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 934 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1345 [chan receive]: +runtime.gopark(0xc002338e18?, 0x7c5b1283c07db?, 0x0?, 0x44?, 0xc000077180?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001488d10 sp=0xc001488cf0 pc=0x4777ae +runtime.chanrecv(0xc00201d500, 0xc001488e40, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001488d88 sp=0xc001488d10 pc=0x441325 +runtime.chanrecv2(0x3b9aca00?, 0x3fa3c40?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc001488db0 sp=0xc001488d88 pc=0x440f92 +github.com/hashicorp/go-plugin.pidWait(0x4b05a9?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/process.go:20 +0xdb fp=0xc001488e68 sp=0xc001488db0 pc=0x216033b +github.com/hashicorp/go-plugin.(*Client).reattach.func1(0x0?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:853 +0xd8 fp=0xc001488fb8 sp=0xc001488e68 pc=0x21528b8 +github.com/hashicorp/go-plugin.(*Client).reattach.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:862 +0x42 fp=0xc001488fe0 sp=0xc001488fb8 pc=0x21527a2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001488fe8 sp=0xc001488fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*Client).reattach in goroutine 1361 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:846 +0x365 + +goroutine 968 [select]: +runtime.gopark(0xc001ef7f20?, 0x2?, 0xb4?, 0x1b?, 0xc001ef7eb4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ef7d28 sp=0xc001ef7d08 pc=0x4777ae +runtime.selectgo(0xc001ef7f20, 0xc001ef7eb0, 0xc001f10480?, 0x0, 0xc001ef7f18?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ef7e78 sp=0xc001ef7d28 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc0019255f0, 0xc001ab75c0, 0xc001ab75d0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001ef7f50 sp=0xc001ef7e78 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeKeyValueStore(0xc0019255f0, {0x5b43370?, 0xc001ab75b0}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:214 +0x1a8 fp=0xc001ef7f98 sp=0xc001ef7f50 pc=0x21a84a8 +github.com/rancher/opni/pkg/gateway.NewGateway.func2.1() + /home/ec2-user/opni/pkg/gateway/gateway.go:157 +0x62 fp=0xc001ef7fe0 sp=0xc001ef7f98 pc=0x22b18c2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ef7fe8 sp=0xc001ef7fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.NewGateway.func2 in goroutine 861 + /home/ec2-user/opni/pkg/gateway/gateway.go:157 +0x3cc + +goroutine 938 [select]: +runtime.gopark(0xc002116890?, 0x2?, 0x98?, 0x67?, 0xc00211683c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0021166a8 sp=0xc002116688 pc=0x4777ae +runtime.selectgo(0xc002116890, 0xc002116838, 0x7f429a465700?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0021167f8 sp=0xc0021166a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc001ed44b0, {0xc001be3db0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0021168c0 sp=0xc0021167f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001ed44b0, {0xc001be3db0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002116958 sp=0xc0021168c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001be3590, {0xc001be3db0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0021169a0 sp=0xc002116958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001be3590}, {0xc001be3db0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0021169e8 sp=0xc0021169a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00123d0e0, {0xc001be3db0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002116a58 sp=0xc0021169e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001be3da0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002116b28 sp=0xc002116a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001ff0680?, 0xc00123d0e0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002116c18 sp=0xc002116b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001be3da0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001ed4be0}, 0xc0010e4480?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002116cd0 sp=0xc002116c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001ec2d20, {0x43dcf60?, 0xc001ed4be0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002116f08 sp=0xc002116cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001be3ef0, {0x43dcf60, 0xc001ed4be0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002116ff0 sp=0xc002116f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc001faa160, {0x43dcf60, 0xc001ed4be0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002117028 sp=0xc002116ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc0018f1b60) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc002117068 sp=0xc002117028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc0018f1b60?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0021170a8 sp=0xc002117068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc0018f1b70) + :1 +0x46 fp=0xc0021170e0 sp=0xc0021170a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0010e4630, {0x5b51368, 0xc0018f1b70}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc002117320 sp=0xc0021170e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00158b250, {0x5b51038, 0xc0018f1b60}) + :1 +0xaf fp=0xc002117370 sp=0xc002117320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00158b250}, {0x5b4a6f0?, 0xc001faa160}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0021173b0 sp=0xc002117370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001be3ef0}, 0xc00149edf8?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002117400 sp=0xc0021173b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001be3ef0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002117490 sp=0xc002117400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001ec2d20}, 0xc00149edf8, 0xc001badc80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0021177d0 sp=0xc002117490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001ec2d20}, 0x3f252a0?, 0xc0018f1a90?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002117848 sp=0xc0021177d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc00123d0e0, 0xc0010e4810, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002117db0 sp=0xc002117848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc00123d0e0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002117f28 sp=0xc002117db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002117fe0 sp=0xc002117f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002117fe8 sp=0xc002117fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 964 [IO wait]: +runtime.gopark(0xc001fb6fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fb6f98 sp=0xc001fb6f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001fb6fd0 sp=0xc001fb6f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163bb38, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001fb6ff0 sp=0xc001fb6fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00182dca0, 0xc002014c00?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001fb7038 sp=0xc001fb6ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00182dc80, {0xc002014c00, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001fb7128 sp=0xc001fb7038 pc=0x50e5a5 +net.(*netFD).Read(0xc00182dc80, {0xc002014c00, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001fb7188 sp=0xc001fb7128 pc=0x89342b +net.(*conn).Read(0xc001ce2978, {0xc002014c00, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001fb7210 sp=0xc001fb7188 pc=0x8b176d +net.(*UnixConn).Read(0xc001ce2978, {0xc002014c00, 0x600, 0x600}) + :1 +0x54 fp=0xc001fb7258 sp=0xc001fb7210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc002e63890, {0xc002014c00, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001fb72b8 sp=0xc001fb7258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc001520d28, {0x5b0f6c0, 0xc002e63890}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc001fb7320 sp=0xc001fb72b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc001520a80, {0x5b0f840?, 0xc001ce2978}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001fb7390 sp=0xc001fb7320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc001520a80, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001fb79c0 sp=0xc001fb7390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc001520a80, {0xc002280000, 0x8000, 0xc002c2b540?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001fb7aa8 sp=0xc001fb79c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001d4d000, {0xc002280000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001fb7af0 sp=0xc001fb7aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc00201d7a0, {0xc000978820, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001fb7bb0 sp=0xc001fb7af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00201d7a0}, {0xc000978820, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001fb7bf8 sp=0xc001fb7bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000978820, 0x9, 0x9}, {0x5b0f0a0, 0xc00201d7a0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001fb7c98 sp=0xc001fb7bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0009787e0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001fb7dd0 sp=0xc001fb7c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000f95d40, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc001fb7fb0 sp=0xc001fb7dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc001fb7fe0 sp=0xc001fb7fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fb7fe8 sp=0xc001fb7fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 883 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 939 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0xe0?, 0x39?, 0x30?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020faf98 sp=0xc0020faf78 pc=0x4777ae +runtime.chanrecv(0xc001bfeae0, 0xc0020fb128, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0020fb010 sp=0xc0020faf98 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0020fb038 sp=0xc0020fb010 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/stream.(*gatewayStreamExtensionServerImpl).ConnectInternal(0xc0007d1b90, {0x5b50ab0, 0xc0018f18f0}) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/stream/plugin_gateway.go:208 +0xa4e fp=0xc0020fb370 sp=0xc0020fb038 pc=0x262088e +github.com/rancher/opni/pkg/plugins/apis/apiextensions._StreamAPIExtension_ConnectInternal_Handler({0x4229080?, 0xc0007d1b90}, {0x5b4a6f0?, 0xc001faa040}) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/apiextensions_grpc.pb.go:284 +0xb6 fp=0xc0020fb3b0 sp=0xc0020fb370 pc=0x218ded6 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x4229080, 0xc0007d1b90}, {0x5b4ab30, 0xc001be38c0}, 0xc00149ec30?, 0x47ee568) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0020fb400 sp=0xc0020fb3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x4229080, 0xc0007d1b90}, {0x5b4ab30, 0xc001be38c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0020fb490 sp=0xc0020fb400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4229080, 0xc0007d1b90}, {0x5b492f8, 0xc001ec2c30}, 0xc00149ec30, 0xc001bad900) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0020fb7d0 sp=0xc0020fb490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x4229080, 0xc0007d1b90}, {0x5b492f8, 0xc001ec2c30}, 0x3f252a0?, 0xc0018f1820?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0020fb848 sp=0xc0020fb7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc00123d320, 0xc0010e4ed0, 0x770c840, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0020fbdb0 sp=0xc0020fb848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc00123d320, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0020fbf28 sp=0xc0020fbdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0020fbfe0 sp=0xc0020fbf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020fbfe8 sp=0xc0020fbfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 940 [select]: +runtime.gopark(0xc00210ffb0?, 0x2?, 0x0?, 0x4e?, 0xc00210ff2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00210fda8 sp=0xc00210fd88 pc=0x4777ae +runtime.selectgo(0xc00210ffb0, 0xc00210ff28, 0x4b0571?, 0x0, 0x11847d3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00210fef8 sp=0xc00210fda8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc00210ffe0 sp=0xc00210fef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00210ffe8 sp=0xc00210ffe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 939 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 941 [select]: +runtime.gopark(0xc0020bef90?, 0x2?, 0xff?, 0xff?, 0xc0020beeac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020bed28 sp=0xc0020bed08 pc=0x4777ae +runtime.selectgo(0xc0020bef90, 0xc0020beea8, 0x5d5276?, 0x0, 0xc001838400?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020bee78 sp=0xc0020bed28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc0015edd40) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc0020befc0 sp=0xc0020bee78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc0020befe0 sp=0xc0020befc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020befe8 sp=0xc0020befe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 939 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 942 [select]: +runtime.gopark(0xc00203df40?, 0x2?, 0xb4?, 0x1b?, 0xc00203decc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029dbd48 sp=0xc0029dbd28 pc=0x4777ae +runtime.selectgo(0xc0029dbf40, 0xc00203dec8, 0xc001be2f00?, 0x0, 0x8ab66c?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029dbe98 sp=0xc0029dbd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc001ed4780}, 0xc0015eddd0, 0x0, 0xc001bfea20, 0xc0018f19b0) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc0029dbf70 sp=0xc0029dbe98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc0029dbfe0 sp=0xc0029dbf70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029dbfe8 sp=0xc0029dbfe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 939 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 943 [chan receive]: +runtime.gopark(0xc001ed4550?, 0x4b05a9?, 0x50?, 0x4e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020c4e20 sp=0xc0020c4e00 pc=0x4777ae +runtime.chanrecv(0xc001be5080, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0020c4e98 sp=0xc0020c4e20 pc=0x441325 +runtime.chanrecv1(0xc001be3890?, 0x2106cfa?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0020c4ec0 sp=0xc0020c4e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc0020c4fe0 sp=0xc0020c4ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020c4fe8 sp=0xc0020c4fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 939 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 944 [chan receive]: +runtime.gopark(0xc001ed4550?, 0x4b05a9?, 0x50?, 0xe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020c0e20 sp=0xc0020c0e00 pc=0x4777ae +runtime.chanrecv(0xc001be5080, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0020c0e98 sp=0xc0020c0e20 pc=0x441325 +runtime.chanrecv1(0xc001be3890?, 0xc001729870?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0020c0ec0 sp=0xc0020c0e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc0020c0fe0 sp=0xc0020c0ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020c0fe8 sp=0xc0020c0fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 939 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 945 [select]: +runtime.gopark(0xc001fc3aa8?, 0x2?, 0x67?, 0x6a?, 0xc001fc39c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fc3838 sp=0xc001fc3818 pc=0x4777ae +runtime.selectgo(0xc001fc3aa8, 0xc001fc39c0, 0xc001fc39d0?, 0x0, 0x43f1c20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001fc3988 sp=0xc001fc3838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc00148edc0) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc001fc3ad8 sp=0xc001fc3988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc00123d440, {0x5b426a8, 0xc001be3890}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc001fc3cf8 sp=0xc001fc3ad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc001fc3fe0 sp=0xc001fc3cf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fc3fe8 sp=0xc001fc3fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 939 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 946 [select]: +runtime.gopark(0xc0020e9710?, 0x2?, 0x0?, 0x0?, 0xc0020e96bc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020e9528 sp=0xc0020e9508 pc=0x4777ae +runtime.selectgo(0xc0020e9710, 0xc0020e96b8, 0x7f429a464200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020e9678 sp=0xc0020e9528 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc001ed45a0, {0xc001be3780, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0020e9740 sp=0xc0020e9678 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001ed45a0, {0xc001be3780, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0020e97d8 sp=0xc0020e9740 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001be36e0, {0xc001be3780, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0020e9820 sp=0xc0020e97d8 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001be36e0}, {0xc001be3780, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0020e9868 sp=0xc0020e9820 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00123d320, {0xc001be3780, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0020e98d8 sp=0xc0020e9868 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001be3770, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0020e99a8 sp=0xc0020e98d8 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc00123d320, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0020e9a98 sp=0xc0020e99a8 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001be3770, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43f1c20, 0xc0023002a0}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0020e9b50 sp=0xc0020e9a98 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001ec2c30, {0x43f1c20?, 0xc0023002a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0020e9d88 sp=0xc0020e9b50 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001be38c0, {0x43f1c20, 0xc0023002a0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0020e9e70 sp=0xc0020e9d88 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc001faa040, {0x43f1c20, 0xc0023002a0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0020e9ea8 sp=0xc0020e9e70 pc=0x1fd94b2 +github.com/rancher/opni/pkg/plugins/apis/apiextensions.(*streamAPIExtensionConnectInternalServer).Recv(0xc0018f18f0) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/apiextensions_grpc.pb.go:303 +0x66 fp=0xc0020e9ee8 sp=0xc0020e9ea8 pc=0x218e046 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc0020e9fe0 sp=0xc0020e9ee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020e9fe8 sp=0xc0020e9fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 945 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1038 [select]: +runtime.gopark(0xc001d75f70?, 0x2?, 0xb4?, 0x1b?, 0xc001d75ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d75d20 sp=0xc001d75d00 pc=0x4777ae +runtime.selectgo(0xc001d75f70, 0xc001d75eb8, 0xc001e6b040?, 0x0, 0x440940?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001d75e70 sp=0xc001d75d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0024893e0, {0x5b426e0, 0xc002055180}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001d75fa0 sp=0xc001d75e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001d75fe0 sp=0xc001d75fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d75fe8 sp=0xc001d75fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 823 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 958 [select]: +runtime.gopark(0xc00210cf90?, 0x2?, 0xff?, 0xff?, 0xc00210ceac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00210cd28 sp=0xc00210cd08 pc=0x4777ae +runtime.selectgo(0xc00210cf90, 0xc00210cea8, 0x0?, 0x0, 0x4b0a55?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00210ce78 sp=0xc00210cd28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc000931200) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc00210cfc0 sp=0xc00210ce78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc00210cfe0 sp=0xc00210cfc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00210cfe8 sp=0xc00210cfe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 956 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 920 [select]: +runtime.gopark(0xc002065f70?, 0x2?, 0xb4?, 0x1b?, 0xc002065ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002065d20 sp=0xc002065d00 pc=0x4777ae +runtime.selectgo(0xc002065f70, 0xc002065eb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002065e70 sp=0xc002065d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b314c0, {0x5b426e0, 0xc001b6e910}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc002065fa0 sp=0xc002065e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc002065fe0 sp=0xc002065fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002065fe8 sp=0xc002065fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 887 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 921 [select]: +runtime.gopark(0xc00205ff70?, 0x2?, 0xb4?, 0x1b?, 0xc00205febc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00205fd20 sp=0xc00205fd00 pc=0x4777ae +runtime.selectgo(0xc00205ff70, 0xc00205feb8, 0x2020200a3736202d?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00205fe70 sp=0xc00205fd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b31540, {0x5b426e0, 0xc001b6e960}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc00205ffa0 sp=0xc00205fe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc00205ffe0 sp=0xc00205ffa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00205ffe8 sp=0xc00205ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 887 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1084 [select]: +runtime.gopark(0xc001151080?, 0x3?, 0xf9?, 0x50?, 0xc00146598a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002937810 sp=0xc0029377f0 pc=0x4777ae +runtime.selectgo(0xc001151080, 0xc001465984, 0x0?, 0x0, 0xc0013404e0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002937960 sp=0xc002937810 pc=0x48916b +reflect.rselect({0xc002937b48, 0x3, 0x4279b60?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:589 +0x345 fp=0xc002937a10 sp=0xc002937960 pc=0x4a8f05 +reflect.Select({0xc000ded180, 0x3, 0x2?}) + /home/ec2-user/sdk/go1.21.0/src/reflect/value.go:3104 +0xb8a fp=0xc002937bd8 sp=0xc002937a10 pc=0x57c44a +github.com/rancher/opni/pkg/util.WaitAll({0x5b426e0, 0xc0012c2730}, 0xc0013403a0, {0xc002937db8, 0x2, 0x2}) + /home/ec2-user/opni/pkg/util/servers.go:51 +0x3ea fp=0xc002937d58 sp=0xc002937bd8 pc=0x176ae8a +github.com/rancher/opni/pkg/management.(*Server).ListenAndServe(0xc0010c0e70, {0x5b426e0, 0xc001fee820}) + /home/ec2-user/opni/pkg/management/server.go:196 +0x208 fp=0xc002937dd8 sp=0xc002937d58 pc=0x231df08 +github.com/rancher/opni/pkg/test.(*Environment).startGateway.func2(0x4b0571) + /home/ec2-user/opni/pkg/test/environment.go:1699 +0x18c fp=0xc002937f88 sp=0xc002937dd8 pc=0x24c574c +github.com/rancher/opni/pkg/plugins/hooks.onLoadingCompletedHook.Invoke(0xc001ded3a0, 0xc001448330?) + /home/ec2-user/opni/pkg/plugins/hooks/complete.go:10 +0x34 fp=0xc002937fa8 sp=0xc002937f88 pc=0x2182b94 +github.com/rancher/opni/pkg/plugins.(*PluginLoader).Complete.func1() + /home/ec2-user/opni/pkg/plugins/loader.go:329 +0x51 fp=0xc002937fe0 sp=0xc002937fa8 pc=0x2189711 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002937fe8 sp=0xc002937fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins.(*PluginLoader).Complete in goroutine 609 + /home/ec2-user/opni/pkg/plugins/loader.go:329 +0x138 + +goroutine 952 [select]: +runtime.gopark(0xc002108fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002108f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002108de8 sp=0xc002108dc8 pc=0x4777ae +runtime.selectgo(0xc002108fb0, 0xc002108f70, 0x4ad421?, 0x0, 0x8d0f0a?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002108f38 sp=0xc002108de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002108fe0 sp=0xc002108f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002108fe8 sp=0xc002108fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 953 [select]: +runtime.gopark(0xc002109fb0?, 0x2?, 0xb0?, 0x9d?, 0xc002109ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002109d58 sp=0xc002109d38 pc=0x4777ae +runtime.selectgo(0xc002109fb0, 0xc002109ee0, 0x1381849?, 0x0, 0xc001729870?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002109ea8 sp=0xc002109d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002109fe0 sp=0xc002109ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002109fe8 sp=0xc002109fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 954 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00210dd60 sp=0xc00210dd40 pc=0x4777ae +runtime.chanrecv(0xc001be5980, 0xc00210de60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00210ddd8 sp=0xc00210dd60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00210de00 sp=0xc00210ddd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc00210dfe0 sp=0xc00210de00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00210dfe8 sp=0xc00210dfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 922 [select]: +runtime.gopark(0xc001e61f70?, 0x2?, 0xb4?, 0x1b?, 0xc001e61ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e61d20 sp=0xc001e61d00 pc=0x4777ae +runtime.selectgo(0xc001e61f70, 0xc001e61eb8, 0x2020200a3736202d?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e61e70 sp=0xc001e61d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b31bc0, {0x5b426e0, 0xc001b6eaf0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001e61fa0 sp=0xc001e61e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001e61fe0 sp=0xc001e61fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e61fe8 sp=0xc001e61fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 886 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 923 [select]: +runtime.gopark(0xc001e5ff70?, 0x2?, 0xb4?, 0x1b?, 0xc001e5febc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e5fd20 sp=0xc001e5fd00 pc=0x4777ae +runtime.selectgo(0xc001e5ff70, 0xc001e5feb8, 0x233731f?, 0x0, 0x472ddb?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e5fe70 sp=0xc001e5fd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b31ce0, {0x5b426e0, 0xc001b6eb40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001e5ffa0 sp=0xc001e5fe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001e5ffe0 sp=0xc001e5ffa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e5ffe8 sp=0xc001e5ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 886 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1040 [select]: +runtime.gopark(0xc002939f70?, 0x2?, 0xb4?, 0x1b?, 0xc002939ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002939d20 sp=0xc002939d00 pc=0x4777ae +runtime.selectgo(0xc002939f70, 0xc002939eb8, 0x0?, 0x0, 0x4aef8c?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002939e70 sp=0xc002939d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0024896a0, {0x5b426e0, 0xc002055450}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc002939fa0 sp=0xc002939e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc002939fe0 sp=0xc002939fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002939fe8 sp=0xc002939fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 823 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1035 [select]: +runtime.gopark(0xc001e3ff80?, 0x3?, 0xb4?, 0x1b?, 0xc001e3fee2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e3fd30 sp=0xc001e3fd10 pc=0x4777ae +runtime.selectgo(0xc001e3ff80, 0xc001e3fedc, 0x4aef8c?, 0x0, 0xc001c65640?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e3fe80 sp=0xc001e3fd30 pc=0x48916b +github.com/lestrrat-go/backoff/v2.(*controller).loop(0xc0022efaa0) + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:52 +0x178 fp=0xc001e3ffc0 sp=0xc001e3fe80 pc=0x13b0398 +github.com/lestrrat-go/backoff/v2.newController.func1() + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x34 fp=0xc001e3ffe0 sp=0xc001e3ffc0 pc=0x13b01f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e3ffe8 sp=0xc001e3ffe0 pc=0x4ad421 +created by github.com/lestrrat-go/backoff/v2.newController in goroutine 1017 + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x4dd + +goroutine 872 [select]: +runtime.gopark(0xc00224be70?, 0x2?, 0x0?, 0x0?, 0xc00224bdf4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00203dc58 sp=0xc00203dc38 pc=0x4777ae +runtime.selectgo(0xc00203de70, 0xc00224bdf0, 0xc00224be00?, 0x0, 0xc00189f500?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00203dda8 sp=0xc00203dc58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001abb6d0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00203dea0 sp=0xc00203dda8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0018b9420) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00203df58 sp=0xc00203dea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc00203dfe0 sp=0xc00203df58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00203dfe8 sp=0xc00203dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 833 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1055 [IO wait]: +runtime.gopark(0xc00215fbf0?, 0x26?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e1fbd0 sp=0xc002e1fbb0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002e1fc08 sp=0xc002e1fbd0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494718, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002e1fc28 sp=0xc002e1fc08 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0025f8920, 0xc002732000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002e1fc70 sp=0xc002e1fc28 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0025f8900, {0xc002732000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc002e1fd60 sp=0xc002e1fc70 pc=0x50e5a5 +net.(*netFD).Read(0xc0025f8900, {0xc002732000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc002e1fdc0 sp=0xc002e1fd60 pc=0x89342b +net.(*conn).Read(0xc001d6fda8, {0xc002732000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc002e1fe48 sp=0xc002e1fdc0 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6fda8, {0xc002732000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc002e1fe90 sp=0xc002e1fe48 pc=0x8d72f4 +github.com/nats-io/nats%2ego.(*natsReader).Read(0xc0027226c0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1890 +0x1a9 fp=0xc002e1ff10 sp=0xc002e1fe90 pc=0x1f6d169 +github.com/nats-io/nats%2ego.(*Conn).readLoop(0xc001658a00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2917 +0x1f8 fp=0xc002e1ffc0 sp=0xc002e1ff10 pc=0x1f75fb8 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x34 fp=0xc002e1ffe0 sp=0xc002e1ffc0 pc=0x1f71514 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e1ffe8 sp=0xc002e1ffe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 823 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x44e + +goroutine 1002 [IO wait]: +runtime.gopark(0xc0021b8fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0021b8f98 sp=0xc0021b8f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0021b8fd0 sp=0xc0021b8f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494de0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0021b8ff0 sp=0xc0021b8fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00193baa0, 0xc00042e800?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0021b9038 sp=0xc0021b8ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00193ba80, {0xc00042e800, 0x3800, 0x3800}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0021b9128 sp=0xc0021b9038 pc=0x50e5a5 +net.(*netFD).Read(0xc00193ba80, {0xc00042e800, 0x3800, 0x3800}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0021b9188 sp=0xc0021b9128 pc=0x89342b +net.(*conn).Read(0xc000991968, {0xc00042e800, 0x3800, 0x3800}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0021b9210 sp=0xc0021b9188 pc=0x8b176d +net.(*UnixConn).Read(0xc000991968, {0xc00042e800, 0x3800, 0x3800}) + :1 +0x54 fp=0xc0021b9258 sp=0xc0021b9210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0030c5b48, {0xc00042e800, 0x3800, 0x3800}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0021b92b8 sp=0xc0021b9258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc001969428, {0x5b0f6c0, 0xc0030c5b48}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0021b9320 sp=0xc0021b92b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc001969180, {0x5b0f840?, 0xc000991968}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc0021b9390 sp=0xc0021b9320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc001969180, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0021b99c0 sp=0xc0021b9390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc001969180, {0xc0021f4000, 0x8000, 0xc003207360?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc0021b9aa8 sp=0xc0021b99c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0021805e0, {0xc0021f4000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc0021b9af0 sp=0xc0021b9aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc0021db020, {0xc0019391c0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0021b9bb0 sp=0xc0021b9af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0021db020}, {0xc0019391c0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0021b9bf8 sp=0xc0021b9bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0019391c0, 0x9, 0x9}, {0x5b0f0a0, 0xc0021db020}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0021b9c98 sp=0xc0021b9bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001939180) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0021b9dd0 sp=0xc0021b9c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001c6db00, 0xc001e6bd40?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0021b9fb0 sp=0xc0021b9dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0021b9fe0 sp=0xc0021b9fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0021b9fe8 sp=0xc0021b9fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 927 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 1447 [select]: +runtime.gopark(0xc002105fb0?, 0x2?, 0xb0?, 0x5d?, 0xc002105ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002105d58 sp=0xc002105d38 pc=0x4777ae +runtime.selectgo(0xc002105fb0, 0xc002105ee0, 0xe?, 0x0, 0x8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002105ea8 sp=0xc002105d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002105fe0 sp=0xc002105ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002105fe8 sp=0xc002105fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 874 [select]: +runtime.gopark(0xc002243e48?, 0x2?, 0x0?, 0x0?, 0xc002243dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020ffc30 sp=0xc0020ffc10 pc=0x4777ae +runtime.selectgo(0xc0020ffe48, 0xc002243dc8, 0xc002243dd8?, 0x0, 0xc0017ca380?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020ffd80 sp=0xc0020ffc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001abb7c0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0020ffe78 sp=0xc0020ffd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0018b9570) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0020fff30 sp=0xc0020ffe78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc0020fffe0 sp=0xc0020fff30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020fffe8 sp=0xc0020fffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 858 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 875 [select]: +runtime.gopark(0xc0017d7f70?, 0x4?, 0x29?, 0x0?, 0xc0017d7dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017d7c00 sp=0xc0017d7be0 pc=0x4777ae +runtime.selectgo(0xc0017d7f70, 0xc0017d7dd0, 0xc0017d7f08?, 0x0, 0x5b50310?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017d7d50 sp=0xc0017d7c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001afda00) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc0017d7fc0 sp=0xc0017d7d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc0017d7fe0 sp=0xc0017d7fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017d7fe8 sp=0xc0017d7fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 858 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 876 [IO wait]: +runtime.gopark(0xc001fc6f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fc6f40 sp=0xc001fc6f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001fc6f78 sp=0xc001fc6f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163be20, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001fc6f98 sp=0xc001fc6f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0017cf7a0, 0xc000d40fc0?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001fc6fe0 sp=0xc001fc6f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0017cf780, {0xc000d40fc0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001fc70d0 sp=0xc001fc6fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc0017cf780, {0xc000d40fc0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001fc7130 sp=0xc001fc70d0 pc=0x89342b +net.(*conn).Read(0xc001d6f4d0, {0xc000d40fc0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001fc71b8 sp=0xc001fc7130 pc=0x8b176d +net.(*UnixConn).Read(0xc001d6f4d0, {0xc000d40fc0, 0x240, 0x240}) + :1 +0x54 fp=0xc001fc7200 sp=0xc001fc71b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc00217fdd0, {0xc000d40fc0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001fc7260 sp=0xc001fc7200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0017ca628, {0x5b0f6c0, 0xc00217fdd0}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc001fc72c8 sp=0xc001fc7260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc0017ca380, {0x5b0f840?, 0xc001d6f4d0}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001fc7338 sp=0xc001fc72c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc0017ca380, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001fc7968 sp=0xc001fc7338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc0017ca380, {0xc002218000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001fc7a50 sp=0xc001fc7968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001d86840, {0xc002218000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001fc7a98 sp=0xc001fc7a50 pc=0x10969ed +bufio.(*Reader).Read(0xc001de63c0, {0xc0011d5a80, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001fc7b58 sp=0xc001fc7a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001de63c0}, {0xc0011d5a80, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001fc7ba0 sp=0xc001fc7b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0011d5a80, 0x9, 0x9}, {0x5b0f0a0, 0xc001de63c0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001fc7c40 sp=0xc001fc7ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011d5a40) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001fc7d78 sp=0xc001fc7c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001afda00, 0x0?, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001fc7ec8 sp=0xc001fc7d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c780, {0x5b54e00?, 0xc001afda00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001fc7f80 sp=0xc001fc7ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001fc7fe0 sp=0xc001fc7f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fc7fe8 sp=0xc001fc7fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 858 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 959 [select]: +runtime.gopark(0xc00212ff40?, 0x2?, 0xb4?, 0x1b?, 0xc00212fecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029e5d48 sp=0xc0029e5d28 pc=0x4777ae +runtime.selectgo(0xc0029e5f40, 0xc00212fec8, 0xc001be2f00?, 0x0, 0x8ab66c?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029e5e98 sp=0xc0029e5d48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc001ed5450}, 0xc000931290, 0x0, 0xc002080cc0, 0xc001c90ba0) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc0029e5f70 sp=0xc0029e5e98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc0029e5fe0 sp=0xc0029e5f70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029e5fe8 sp=0xc0029e5fe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 956 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 960 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc001be4e40?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00213ae20 sp=0xc00213ae00 pc=0x4777ae +runtime.chanrecv(0xc001be4e40, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00213ae98 sp=0xc00213ae20 pc=0x441325 +runtime.chanrecv1(0xc001be3350?, 0x8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00213aec0 sp=0xc00213ae98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc00213afe0 sp=0xc00213aec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00213afe8 sp=0xc00213afe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 956 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 961 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc001be4e40?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002163e20 sp=0xc002163e00 pc=0x4777ae +runtime.chanrecv(0xc001be4e40, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002163e98 sp=0xc002163e20 pc=0x441325 +runtime.chanrecv1(0xc001be3350?, 0xc000086ee8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002163ec0 sp=0xc002163e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002163fe0 sp=0xc002163ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002163fe8 sp=0xc002163fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 956 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 978 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc001be4e40?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00213ce20 sp=0xc00213ce00 pc=0x4777ae +runtime.chanrecv(0xc001be4e40, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00213ce98 sp=0xc00213ce20 pc=0x441325 +runtime.chanrecv1(0xc001be3350?, 0x43148a0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00213cec0 sp=0xc00213ce98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc00213cfe0 sp=0xc00213cec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00213cfe8 sp=0xc00213cfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 956 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 979 [select]: +runtime.gopark(0xc001d7baa8?, 0x2?, 0x67?, 0x6a?, 0xc001d7b9c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d7b838 sp=0xc001d7b818 pc=0x4777ae +runtime.selectgo(0xc001d7baa8, 0xc001d7b9c0, 0xc001d7b9d0?, 0x0, 0x43f1c20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001d7b988 sp=0xc001d7b838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc00148f0e0) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc001d7bad8 sp=0xc001d7b988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc00123dc20, {0x5b426a8, 0xc001be3350}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc001d7bcf8 sp=0xc001d7bad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc001d7bfe0 sp=0xc001d7bcf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d7bfe8 sp=0xc001d7bfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 956 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 980 [select]: +runtime.gopark(0xc0020eb3f8?, 0x2?, 0xc0?, 0x72?, 0xc0020eb37c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020eb1e8 sp=0xc0020eb1c8 pc=0x4777ae +runtime.selectgo(0xc0020eb3f8, 0xc0020eb378, 0x5b203e8?, 0x0, 0x2710?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020eb338 sp=0xc0020eb1e8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc001ed4370, {0xc001be3420, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc0020eb428 sp=0xc0020eb338 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001ed4370, {0xc001be3420, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc0020eb4c0 sp=0xc0020eb428 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001be33e0, {0xc001be3420, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0020eb508 sp=0xc0020eb4c0 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001be33e0}, {0xc001be3420, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0020eb550 sp=0xc0020eb508 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00123cfc0, {0xc001be3420, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0020eb5c0 sp=0xc0020eb550 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001be3410, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0020eb690 sp=0xc0020eb5c0 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc00123cfc0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0020eb780 sp=0xc0020eb690 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001be3410, {0x7f424347ae80, 0x779c980}, 0x448485?, {0x0, 0x0}, {0x43f1c20, 0xc002300690}, 0x800?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0020eb838 sp=0xc0020eb780 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bd1930, {0x43f1c20?, 0xc002300690}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc0020eba98 sp=0xc0020eb838 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00123ce34?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc0020ebaf8 sp=0xc0020eba98 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00123cd80, 0xc0020ebcb0, 0xc0020ebc80) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc0020ebba0 sp=0xc0020ebaf8 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00123cd80, {0x43f1c20, 0xc002300690}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc0020ebce0 sp=0xc0020ebba0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc001bad780, {0x43f1c20, 0xc002300690}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc0020ebea8 sp=0xc0020ebce0 pc=0x12ac5b9 +github.com/rancher/opni/pkg/plugins/apis/apiextensions.(*streamAPIExtensionConnectInternalClient).Recv(0xc0018f16e0) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/apiextensions_grpc.pb.go:249 +0x66 fp=0xc0020ebee8 sp=0xc0020ebea8 pc=0x218dd66 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc0020ebfe0 sp=0xc0020ebee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020ebfe8 sp=0xc0020ebfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 979 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 982 [IO wait]: +runtime.gopark(0xc002234fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002234f98 sp=0xc002234f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002234fd0 sp=0xc002234f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163bd28, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002234ff0 sp=0xc002234fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001ab0920, 0xc002130600?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002235038 sp=0xc002234ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001ab0900, {0xc002130600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc002235128 sp=0xc002235038 pc=0x50e5a5 +net.(*netFD).Read(0xc001ab0900, {0xc002130600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc002235188 sp=0xc002235128 pc=0x89342b +net.(*conn).Read(0xc000b79f30, {0xc002130600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc002235210 sp=0xc002235188 pc=0x8b176d +net.(*UnixConn).Read(0xc000b79f30, {0xc002130600, 0x600, 0x600}) + :1 +0x54 fp=0xc002235258 sp=0xc002235210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc00217e4c8, {0xc002130600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0022352b8 sp=0xc002235258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc00189fb28, {0x5b0f6c0, 0xc00217e4c8}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc002235320 sp=0xc0022352b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc00189f880, {0x5b0f840?, 0xc000b79f30}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc002235390 sp=0xc002235320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc00189f880, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0022359c0 sp=0xc002235390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc00189f880, {0xc00213e000, 0x8000, 0xc002235b38?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc002235aa8 sp=0xc0022359c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001faa600, {0xc00213e000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc002235af0 sp=0xc002235aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc002081260, {0xc000881700, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc002235bb0 sp=0xc002235af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc002081260}, {0xc000881700, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002235bf8 sp=0xc002235bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000881700, 0x9, 0x9}, {0x5b0f0a0, 0xc002081260}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc002235c98 sp=0xc002235bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0008816c0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc002235dd0 sp=0xc002235c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001bfc900, 0xc0020812c0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc002235fb0 sp=0xc002235dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc002235fe0 sp=0xc002235fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002235fe8 sp=0xc002235fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 868 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 965 [select]: +runtime.gopark(0xc003025e70?, 0x2?, 0x0?, 0x0?, 0xc003025df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc003025c58 sp=0xc003025c38 pc=0x4777ae +runtime.selectgo(0xc003025e70, 0xc003025df0, 0xc002006e70?, 0x0, 0xc001520a80?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc003025da8 sp=0xc003025c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001c65c20, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc003025ea0 sp=0xc003025da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002006e70) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc003025f58 sp=0xc003025ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc003025fe0 sp=0xc003025f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc003025fe8 sp=0xc003025fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 883 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 983 [select]: +runtime.gopark(0xc00216fe70?, 0x2?, 0x0?, 0x0?, 0xc00216fdf4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e1fc58 sp=0xc001e1fc38 pc=0x4777ae +runtime.selectgo(0xc001e1fe70, 0xc00216fdf0, 0xc00216fe00?, 0x0, 0xc00189f880?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e1fda8 sp=0xc001e1fc58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001ed57c0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001e1fea0 sp=0xc001e1fda8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0017dbc00) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001e1ff58 sp=0xc001e1fea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc001e1ffe0 sp=0xc001e1ff58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e1ffe8 sp=0xc001e1ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 868 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1017 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x30?, 0x15?, 0x10?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001eeacf8 sp=0xc001eeacd8 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001eead70 sp=0xc001eeacf8 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0x5b55580?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001eead98 sp=0xc001eead70 pc=0x440f72 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseKeyValueStore(0xc00070a000, {0x5b43f90?, 0xc00234e480}) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:185 +0xc25 fp=0xc001eeafd0 sp=0xc001eead98 pc=0x3970905 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseKeyValueStore(0xc001367cc0, {0x4b05a9?, 0xc0023350f0?}, 0xc0023262a0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:123 +0x18d fp=0xc001eeb098 sp=0xc001eeafd0 pc=0x21a6c2d +github.com/rancher/opni/pkg/plugins/apis/system._System_UseKeyValueStore_Handler.func1({0x5b426a8, 0xc002326360}, {0x4278de0?, 0xc0023262a0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:187 +0x83 fp=0xc001eeb0f0 sp=0xc001eeb098 pc=0x21ac823 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc002326360}, {0x4278de0, 0xc0023262a0}, 0xc001fab480, 0xc00217e6a8) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc001eeb148 sp=0xc001eeb0f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc002326360}, {0x4278de0, 0xc0023262a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc001eeb1e8 sp=0xc001eeb148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc002326270}, {0x4278de0, 0xc0023262a0}, 0xc001fab480, 0xc002170d80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc001eeb6b8 sp=0xc001eeb1e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc002326270}, {0x4278de0, 0xc0023262a0}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc001eeb740 sp=0xc001eeb6b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseKeyValueStore_Handler({0x4278ea0?, 0xc001367cc0}, {0x5b426a8, 0xc002326270}, 0xc002300620, 0xc000ee6aa0) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:189 +0x1c4 fp=0xc001eeb7d0 sp=0xc001eeb740 pc=0x21ac684 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001335440, 0xc001267440, 0x771ecd0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc001eebdb0 sp=0xc001eeb7d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001335440, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc001eebf28 sp=0xc001eebdb0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001eebfe0 sp=0xc001eebf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001eebfe8 sp=0xc001eebfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1014 [select]: +runtime.gopark(0xc00232fe48?, 0x2?, 0x0?, 0x0?, 0xc00232fdcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002123c30 sp=0xc002123c10 pc=0x4777ae +runtime.selectgo(0xc002123e48, 0xc00232fdc8, 0xc00232fdd8?, 0x0, 0xc001520e00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002123d80 sp=0xc002123c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002167bd0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002123e78 sp=0xc002123d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0023005b0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002123f30 sp=0xc002123e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc002123fe0 sp=0xc002123f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002123fe8 sp=0xc002123fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 966 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1454 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e1acd0 sp=0xc001e1acb0 pc=0x4777ae +runtime.chanrecv(0xc002d20240, 0xc001e1af90, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001e1ad48 sp=0xc001e1acd0 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001e1ad70 sp=0xc001e1ad48 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/stream.(*agentStreamExtensionServerImpl).Connect(0xc001e54e40, {0x5b508a0, 0xc002d00490}) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/stream/plugin_agent.go:166 +0x1065 fp=0xc001e1b370 sp=0xc001e1ad70 pc=0x261cb65 +github.com/rancher/opni/pkg/apis/stream/v1._Stream_Connect_Handler({0x4229120?, 0xc001e54e40}, {0x5b4a6f0?, 0xc002488b60}) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:121 +0xb6 fp=0xc001e1b3b0 sp=0xc001e1b370 pc=0x2003ed6 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x4229120, 0xc001e54e40}, {0x5b4ab30, 0xc00266d9e0}, 0xc0012e4f18?, 0x47ee1d0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001e1b400 sp=0xc001e1b3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x4229120, 0xc001e54e40}, {0x5b4ab30, 0xc00266d9e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001e1b490 sp=0xc001e1b400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4229120, 0xc001e54e40}, {0x5b492f8, 0xc000b9c2d0}, 0xc0012e4f18, 0xc001d7de40) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001e1b7d0 sp=0xc001e1b490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x4229120, 0xc001e54e40}, {0x5b492f8, 0xc000b9c2d0}, 0x3f252a0?, 0xc002d001f0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001e1b848 sp=0xc001e1b7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c58360, 0xc0025cd6b0, 0x770c800, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001e1bdb0 sp=0xc001e1b848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c58360, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001e1bf28 sp=0xc001e1bdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001e1bfe0 sp=0xc001e1bf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e1bfe8 sp=0xc001e1bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1387 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 988 [select]: +runtime.gopark(0xc0017d8fb0?, 0x2?, 0xb4?, 0x1b?, 0xc0017d8f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017d8de8 sp=0xc0017d8dc8 pc=0x4777ae +runtime.selectgo(0xc0017d8fb0, 0xc0017d8f70, 0x4ad421?, 0x0, 0x8d0f0a?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017d8f38 sp=0xc0017d8de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0017d8fe0 sp=0xc0017d8f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017d8fe8 sp=0xc0017d8fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 989 [select]: +runtime.gopark(0xc00213bfb0?, 0x2?, 0xb0?, 0xbd?, 0xc00213bee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00213bd58 sp=0xc00213bd38 pc=0x4777ae +runtime.selectgo(0xc00213bfb0, 0xc00213bee0, 0xf?, 0x0, 0x8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00213bea8 sp=0xc00213bd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc00213bfe0 sp=0xc00213bea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00213bfe8 sp=0xc00213bfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 990 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002139d60 sp=0xc002139d40 pc=0x4777ae +runtime.chanrecv(0xc002156600, 0xc002139e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002139dd8 sp=0xc002139d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002139e00 sp=0xc002139dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002139fe0 sp=0xc002139e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002139fe8 sp=0xc002139fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 994 [select]: +runtime.gopark(0xc002034890?, 0x2?, 0x98?, 0x47?, 0xc00203483c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020346a8 sp=0xc002034688 pc=0x4777ae +runtime.selectgo(0xc002034890, 0xc002034838, 0x7f429a45f700?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020347f8 sp=0xc0020346a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc001b6f4f0, {0xc0021ba9d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0020348c0 sp=0xc0020347f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001b6f4f0, {0xc0021ba9d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002034958 sp=0xc0020348c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001b41c50, {0xc0021ba9d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0020349a0 sp=0xc002034958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001b41c50}, {0xc0021ba9d0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0020349e8 sp=0xc0020349a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00146a7e0, {0xc0021ba9d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002034a58 sp=0xc0020349e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0021ba9c0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002034b28 sp=0xc002034a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001e41840?, 0xc00146a7e0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002034c18 sp=0xc002034b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0021ba9c0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001b6fc70}, 0xc001266cc0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002034cd0 sp=0xc002034c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001b9c960, {0x43dcf60?, 0xc001b6fc70}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002034f08 sp=0xc002034cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0021bab10, {0x43dcf60, 0xc001b6fc70}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002034ff0 sp=0xc002034f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002180520, {0x43dcf60, 0xc001b6fc70}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002035028 sp=0xc002034ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc001d2c210) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc002035068 sp=0xc002035028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc001d2c210?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0020350a8 sp=0xc002035068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc001d2c220) + :1 +0x46 fp=0xc0020350e0 sp=0xc0020350a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc001266ea0, {0x5b51368, 0xc001d2c220}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc002035320 sp=0xc0020350e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc001510830, {0x5b51038, 0xc001d2c210}) + :1 +0xaf fp=0xc002035370 sp=0xc002035320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc001510830}, {0x5b4a6f0?, 0xc002180520}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0020353b0 sp=0xc002035370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc0021bab10}, 0xc001e46828?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002035400 sp=0xc0020353b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc0021bab10}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002035490 sp=0xc002035400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001b9c960}, 0xc001e46828, 0xc001e40c40) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0020357d0 sp=0xc002035490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001b9c960}, 0x3f252a0?, 0xc001d2c130?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002035848 sp=0xc0020357d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00146a7e0, 0xc001266f30, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002035db0 sp=0xc002035848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00146a7e0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002035f28 sp=0xc002035db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002035fe0 sp=0xc002035f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002035fe8 sp=0xc002035fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 995 [select]: +runtime.gopark(0xc0021e5e48?, 0x2?, 0x0?, 0x0?, 0xc0021e5dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001551c30 sp=0xc001551c10 pc=0x4777ae +runtime.selectgo(0xc001551e48, 0xc0021e5dc8, 0xc0021e5dd8?, 0x0, 0xc001968a80?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001551d80 sp=0xc001551c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001b6f590, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001551e78 sp=0xc001551d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001b3fe30) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001551f30 sp=0xc001551e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc001551fe0 sp=0xc001551f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001551fe8 sp=0xc001551fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 917 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 996 [select]: +runtime.gopark(0xc0020bff70?, 0x4?, 0x20?, 0xbc?, 0xc0020bfdd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020bfc00 sp=0xc0020bfbe0 pc=0x4777ae +runtime.selectgo(0xc0020bff70, 0xc0020bfdd0, 0xb000000004b0571?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020bfd50 sp=0xc0020bfc00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001e6b6c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc0020bffc0 sp=0xc0020bfd50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc0020bffe0 sp=0xc0020bffc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020bffe8 sp=0xc0020bffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 917 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 997 [IO wait]: +runtime.gopark(0xc0021b6f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0021b6f40 sp=0xc0021b6f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0021b6f78 sp=0xc0021b6f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163b758, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0021b6f98 sp=0xc0021b6f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00193b5a0, 0xc00047f440?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0021b6fe0 sp=0xc0021b6f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00193b580, {0xc00047f440, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0021b70d0 sp=0xc0021b6fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc00193b580, {0xc00047f440, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0021b7130 sp=0xc0021b70d0 pc=0x89342b +net.(*conn).Read(0xc000991868, {0xc00047f440, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0021b71b8 sp=0xc0021b7130 pc=0x8b176d +net.(*UnixConn).Read(0xc000991868, {0xc00047f440, 0x240, 0x240}) + :1 +0x54 fp=0xc0021b7200 sp=0xc0021b71b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc00217e120, {0xc00047f440, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0021b7260 sp=0xc0021b7200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc001968d28, {0x5b0f6c0, 0xc00217e120}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0021b72c8 sp=0xc0021b7260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc001968a80, {0x5b0f840?, 0xc000991868}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc0021b7338 sp=0xc0021b72c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc001968a80, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0021b7968 sp=0xc0021b7338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc001968a80, {0xc00219a000, 0x8000, 0x0?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc0021b7a50 sp=0xc0021b7968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0021802c0, {0xc00219a000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc0021b7a98 sp=0xc0021b7a50 pc=0x10969ed +bufio.(*Reader).Read(0xc002194f60, {0xc001939000, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0021b7b58 sp=0xc0021b7a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc002194f60}, {0xc001939000, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0021b7ba0 sp=0xc0021b7b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc001939000, 0x9, 0x9}, {0x5b0f0a0, 0xc002194f60}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0021b7c40 sp=0xc0021b7ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001938fc0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0021b7d78 sp=0xc0021b7c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001e6b6c0, 0x313231202d000000?, 0xc000f3de70?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc0021b7ec8 sp=0xc0021b7d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c960, {0x5b54e00?, 0xc001e6b6c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc0021b7f80 sp=0xc0021b7ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc0021b7fe0 sp=0xc0021b7f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0021b7fe8 sp=0xc0021b7fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 917 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 999 [select]: +runtime.gopark(0xc00302de48?, 0x2?, 0x0?, 0x0?, 0xc00302ddcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00302dc30 sp=0xc00302dc10 pc=0x4777ae +runtime.selectgo(0xc00302de48, 0xc00302ddc8, 0xc00302ddd8?, 0x0, 0xc001968700?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00302dd80 sp=0xc00302dc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001b6f8b0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00302de78 sp=0xc00302dd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001b3fea0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00302df30 sp=0xc00302de78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc00302dfe0 sp=0xc00302df30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00302dfe8 sp=0xc00302dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 849 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1000 [select]: +runtime.gopark(0xc001ef5f70?, 0x4?, 0x78?, 0xf8?, 0xc001ef5dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ef5c00 sp=0xc001ef5be0 pc=0x4777ae +runtime.selectgo(0xc001ef5f70, 0xc001ef5dd0, 0x40a16a0?, 0x0, 0xc001d2c040?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ef5d50 sp=0xc001ef5c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001e6bd40) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc001ef5fc0 sp=0xc001ef5d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc001ef5fe0 sp=0xc001ef5fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ef5fe8 sp=0xc001ef5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 849 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1001 [IO wait]: +runtime.gopark(0xc001f42f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f42f40 sp=0xc001f42f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001f42f78 sp=0xc001f42f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163b850, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001f42f98 sp=0xc001f42f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00193b420, 0xc00047efc0?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001f42fe0 sp=0xc001f42f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00193b400, {0xc00047efc0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001f430d0 sp=0xc001f42fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc00193b400, {0xc00047efc0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001f43130 sp=0xc001f430d0 pc=0x89342b +net.(*conn).Read(0xc000991860, {0xc00047efc0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001f431b8 sp=0xc001f43130 pc=0x8b176d +net.(*UnixConn).Read(0xc000991860, {0xc00047efc0, 0x240, 0x240}) + :1 +0x54 fp=0xc001f43200 sp=0xc001f431b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0030c4138, {0xc00047efc0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001f43260 sp=0xc001f43200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0019689a8, {0x5b0f6c0, 0xc0030c4138}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc001f432c8 sp=0xc001f43260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc001968700, {0x5b0f840?, 0xc000991860}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001f43338 sp=0xc001f432c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc001968700, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001f43968 sp=0xc001f43338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc001968700, {0xc0021bc000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001f43a50 sp=0xc001f43968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0021803c0, {0xc0021bc000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001f43a98 sp=0xc001f43a50 pc=0x10969ed +bufio.(*Reader).Read(0xc0021953e0, {0xc0019390e0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001f43b58 sp=0xc001f43a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0021953e0}, {0xc0019390e0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001f43ba0 sp=0xc001f43b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0019390e0, 0x9, 0x9}, {0x5b0f0a0, 0xc0021953e0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001f43c40 sp=0xc001f43ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0019390a0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001f43d78 sp=0xc001f43c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001e6bd40, 0x4aead9?, 0x4b0571?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001f43ec8 sp=0xc001f43d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc0018525a0, {0x5b54e00?, 0xc001e6bd40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001f43f80 sp=0xc001f43ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001f43fe0 sp=0xc001f43f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f43fe8 sp=0xc001f43fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 849 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 972 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0021b3978 sp=0xc0021b3958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0021b39b0 sp=0xc0021b3978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163b568, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0021b39d0 sp=0xc0021b39b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001f00520, 0xc0021b3a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0021b3a18 sp=0xc0021b39d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc001f00500) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0021b3b08 sp=0xc0021b3a18 pc=0x513f65 +net.(*netFD).accept(0xc001f00500) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0021b3c00 sp=0xc0021b3b08 pc=0x8967be +net.(*UnixListener).accept(0xc002008cc0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc0021b3c40 sp=0xc0021b3c00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc002008cc0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc0021b3cb0 sp=0xc0021b3c40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc001d4d260) + :1 +0x44 fp=0xc0021b3ce8 sp=0xc0021b3cb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc001852b40, {0x5b2c6a0, 0xc001d4d260}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc0021b3f18 sp=0xc0021b3ce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc0021b3f68 sp=0xc0021b3f18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc001d4d480?, 0xc001ab77d0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc0021b3fa8 sp=0xc0021b3f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc0021b3fe0 sp=0xc0021b3fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0021b3fe8 sp=0xc0021b3fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 969 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 973 [select]: +runtime.gopark(0xc001e39f38?, 0x2?, 0x0?, 0x0?, 0xc001e39f14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e39d90 sp=0xc001e39d70 pc=0x4777ae +runtime.selectgo(0xc001e39f38, 0xc001e39f10, 0xc001e39f18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e39ee0 sp=0xc001e39d90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc001e39f68 sp=0xc001e39ee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc000dc2cd8?, 0xc001ab77f0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001e39fa8 sp=0xc001e39f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001e39fe0 sp=0xc001e39fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e39fe8 sp=0xc001e39fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 969 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 1011 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0021368d8 sp=0xc0021368b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002136910 sp=0xc0021368d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494a00, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002136930 sp=0xc002136910 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001e9f020, 0xc001ff7701?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002136978 sp=0xc002136930 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc001e9f000) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc002136a68 sp=0xc002136978 pc=0x513f65 +net.(*netFD).accept(0xc001e9f000) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc002136b60 sp=0xc002136a68 pc=0x8967be +net.(*TCPListener).accept(0xc001faac20) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc002136ba8 sp=0xc002136b60 pc=0x8c383e +net.(*TCPListener).Accept(0xc001faac20) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc002136c18 sp=0xc002136ba8 pc=0x8c1ba5 +crypto/tls.(*listener).Accept(0xc00149fec0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/tls.go:66 +0x43 fp=0xc002136c70 sp=0xc002136c18 pc=0x9b2d43 +net/http.(*onceCloseListener).Accept(0xc002173710) + :1 +0x44 fp=0xc002136ca8 sp=0xc002136c70 pc=0xab5544 +net/http.(*Server).Serve(0xc001ec2e10, {0x5b2a5b0, 0xc00149fec0}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc002136ea0 sp=0xc002136ca8 pc=0xa7859f +net/http.Serve(...) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:2595 +github.com/gin-gonic/gin.(*Engine).RunListener(0xc0018ac680, {0x5b2a5b0, 0xc00149fec0}) + /home/ec2-user/go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/gin.go:565 +0x370 fp=0xc002136f90 sp=0xc002136ea0 pc=0x171e870 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure.func1() + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:72 +0x5c fp=0xc002136fe0 sp=0xc002136f90 pc=0x262a57c +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002136fe8 sp=0xc002136fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure in goroutine 1010 + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:71 +0x26d + +goroutine 976 [IO wait]: +runtime.gopark(0xc0021b4fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0021b4f98 sp=0xc0021b4f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0021b4fd0 sp=0xc0021b4f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163b660, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0021b4ff0 sp=0xc0021b4fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00193b8a0, 0xc0022f8000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0021b5038 sp=0xc0021b4ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00193b880, {0xc0022f8000, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0021b5128 sp=0xc0021b5038 pc=0x50e5a5 +net.(*netFD).Read(0xc00193b880, {0xc0022f8000, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0021b5188 sp=0xc0021b5128 pc=0x89342b +net.(*conn).Read(0xc000991960, {0xc0022f8000, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0021b5210 sp=0xc0021b5188 pc=0x8b176d +net.(*UnixConn).Read(0xc000991960, {0xc0022f8000, 0x600, 0x600}) + :1 +0x54 fp=0xc0021b5258 sp=0xc0021b5210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc000dc2f90, {0xc0022f8000, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0021b52b8 sp=0xc0021b5258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0019690a8, {0x5b0f6c0, 0xc000dc2f90}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0021b5320 sp=0xc0021b52b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc001968e00, {0x5b0f840?, 0xc000991960}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc0021b5390 sp=0xc0021b5320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc001968e00, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0021b59c0 sp=0xc0021b5390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc001968e00, {0xc0023da000, 0x8000, 0x4?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc0021b5aa8 sp=0xc0021b59c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001d4d6a0, {0xc0023da000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc0021b5af0 sp=0xc0021b5aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc0022eee40, {0xc000978900, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0021b5bb0 sp=0xc0021b5af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0022eee40}, {0xc000978900, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0021b5bf8 sp=0xc0021b5bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000978900, 0x9, 0x9}, {0x5b0f0a0, 0xc0022eee40}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0021b5c98 sp=0xc0021b5bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0009788c0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0021b5dd0 sp=0xc0021b5c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc002028480, 0xc0022eeea0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0021b5fb0 sp=0xc0021b5dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0021b5fe0 sp=0xc0021b5fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0021b5fe8 sp=0xc0021b5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 924 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 977 [select]: +runtime.gopark(0xc002423e70?, 0x2?, 0x0?, 0x0?, 0xc002423df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001229c58 sp=0xc001229c38 pc=0x4777ae +runtime.selectgo(0xc001229e70, 0xc002423df0, 0xc002423e00?, 0x0, 0xc001968e00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001229da8 sp=0xc001229c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002054460, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001229ea0 sp=0xc001229da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002007dc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001229f58 sp=0xc001229ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc001229fe0 sp=0xc001229f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001229fe8 sp=0xc001229fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 924 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1041 [select]: +runtime.gopark(0xc0029e3f70?, 0x2?, 0xb4?, 0x1b?, 0xc0029e3ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029e3d20 sp=0xc0029e3d00 pc=0x4777ae +runtime.selectgo(0xc0029e3f70, 0xc0029e3eb8, 0xc001e6ad00?, 0x0, 0x440940?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029e3e70 sp=0xc0029e3d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002489700, {0x5b426e0, 0xc0020554a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0029e3fa0 sp=0xc0029e3e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0029e3fe0 sp=0xc0029e3fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029e3fe8 sp=0xc0029e3fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 823 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1006 [select]: +runtime.gopark(0xc002160f70?, 0x4?, 0x58?, 0xc?, 0xc002160dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002160c00 sp=0xc002160be0 pc=0x4777ae +runtime.selectgo(0xc002160f70, 0xc002160dd0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002160d50 sp=0xc002160c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0021d44e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc002160fc0 sp=0xc002160d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc002160fe0 sp=0xc002160fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002160fe8 sp=0xc002160fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 873 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1007 [IO wait]: +runtime.gopark(0xc002330f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002330f40 sp=0xc002330f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002330f78 sp=0xc002330f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494ce8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002330f98 sp=0xc002330f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001ab0c20, 0xc0015dd8c0?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002330fe0 sp=0xc002330f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001ab0c00, {0xc0015dd8c0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0023310d0 sp=0xc002330fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc001ab0c00, {0xc0015dd8c0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc002331130 sp=0xc0023310d0 pc=0x89342b +net.(*conn).Read(0xc000b79f88, {0xc0015dd8c0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0023311b8 sp=0xc002331130 pc=0x8b176d +net.(*UnixConn).Read(0xc000b79f88, {0xc0015dd8c0, 0x240, 0x240}) + :1 +0x54 fp=0xc002331200 sp=0xc0023311b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc00304b620, {0xc0015dd8c0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc002331260 sp=0xc002331200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc00189fea8, {0x5b0f6c0, 0xc00304b620}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0023312c8 sp=0xc002331260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc00189fc00, {0x5b0f840?, 0xc000b79f88}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc002331338 sp=0xc0023312c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc00189fc00, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc002331968 sp=0xc002331338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc00189fc00, {0xc0023f6000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc002331a50 sp=0xc002331968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc002180740, {0xc0023f6000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc002331a98 sp=0xc002331a50 pc=0x10969ed +bufio.(*Reader).Read(0xc0021db500, {0xc0019392a0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc002331b58 sp=0xc002331a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0021db500}, {0xc0019392a0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002331ba0 sp=0xc002331b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0019392a0, 0x9, 0x9}, {0x5b0f0a0, 0xc0021db500}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc002331c40 sp=0xc002331ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001939260) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc002331d78 sp=0xc002331c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0021d44e0, 0x616d656863732422?, 0x3a70747468223a22?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc002331ec8 sp=0xc002331d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc001ebfc20, {0x5b54e00?, 0xc0021d44e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc002331f80 sp=0xc002331ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc002331fe0 sp=0xc002331f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002331fe8 sp=0xc002331fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 873 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1018 [select]: +runtime.gopark(0xc002403fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002403f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002403de8 sp=0xc002403dc8 pc=0x4777ae +runtime.selectgo(0xc002403fb0, 0xc002403f70, 0xc0021565a0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002403f38 sp=0xc002403de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002403fe0 sp=0xc002403f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002403fe8 sp=0xc002403fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 992 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1019 [select]: +runtime.gopark(0xc002138fb0?, 0x2?, 0xb0?, 0x8d?, 0xc002138ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002138d58 sp=0xc002138d38 pc=0x4777ae +runtime.selectgo(0xc002138fb0, 0xc002138ee0, 0x8ab66c?, 0x0, 0x4b0571?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002138ea8 sp=0xc002138d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002138fe0 sp=0xc002138ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002138fe8 sp=0xc002138fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 992 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1020 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002404d60 sp=0xc002404d40 pc=0x4777ae +runtime.chanrecv(0xc0021575c0, 0xc002404e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002404dd8 sp=0xc002404d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002404e00 sp=0xc002404dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002404fe0 sp=0xc002404e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002404fe8 sp=0xc002404fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 992 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1102 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x14?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018e1bd0 sp=0xc0018e1bb0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0018e1c08 sp=0xc0018e1bd0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494620, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0018e1c28 sp=0xc0018e1c08 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc002a289a0, 0xc002a2a000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0018e1c70 sp=0xc0018e1c28 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc002a28980, {0xc002a2a000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0018e1d60 sp=0xc0018e1c70 pc=0x50e5a5 +net.(*netFD).Read(0xc002a28980, {0xc002a2a000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0018e1dc0 sp=0xc0018e1d60 pc=0x89342b +net.(*conn).Read(0xc00007d6f0, {0xc002a2a000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0018e1e48 sp=0xc0018e1dc0 pc=0x8b176d +net.(*TCPConn).Read(0xc00007d6f0, {0xc002a2a000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc0018e1e90 sp=0xc0018e1e48 pc=0x8d72f4 +github.com/nats-io/nats%2ego.(*natsReader).Read(0xc002a20440) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1890 +0x1a9 fp=0xc0018e1f10 sp=0xc0018e1e90 pc=0x1f6d169 +github.com/nats-io/nats%2ego.(*Conn).readLoop(0xc001659100) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2917 +0x1f8 fp=0xc0018e1fc0 sp=0xc0018e1f10 pc=0x1f75fb8 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x34 fp=0xc0018e1fe0 sp=0xc0018e1fc0 pc=0x1f71514 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018e1fe8 sp=0xc0018e1fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 1017 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x44e + +goroutine 1022 [select]: +runtime.gopark(0xc002406fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002406f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002406de8 sp=0xc002406dc8 pc=0x4777ae +runtime.selectgo(0xc002406fb0, 0xc002406f70, 0xc001be5920?, 0x0, 0xc001fe4540?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002406f38 sp=0xc002406de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002406fe0 sp=0xc002406f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002406fe8 sp=0xc002406fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1023 [select]: +runtime.gopark(0xc00234dfb0?, 0x2?, 0xb0?, 0xdd?, 0xc00234dee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00234dd58 sp=0xc00234dd38 pc=0x4777ae +runtime.selectgo(0xc00234dfb0, 0xc00234dee0, 0x1381849?, 0x0, 0xc000086ee8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00234dea8 sp=0xc00234dd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc00234dfe0 sp=0xc00234dea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00234dfe8 sp=0xc00234dfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1024 [chan receive]: +runtime.gopark(0xc000881b40?, 0xc002137eb8?, 0xbb?, 0xe2?, 0xc000881b20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002137d60 sp=0xc002137d40 pc=0x4777ae +runtime.chanrecv(0xc002157860, 0xc002137e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002137dd8 sp=0xc002137d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002137e00 sp=0xc002137dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002137fe0 sp=0xc002137e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002137fe8 sp=0xc002137fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1106 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x26?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e38bd0 sp=0xc001e38bb0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001e38c08 sp=0xc001e38bd0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494810, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001e38c28 sp=0xc001e38c08 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001f01820, 0xc00291e000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001e38c70 sp=0xc001e38c28 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001f01800, {0xc00291e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001e38d60 sp=0xc001e38c70 pc=0x50e5a5 +net.(*netFD).Read(0xc001f01800, {0xc00291e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001e38dc0 sp=0xc001e38d60 pc=0x89342b +net.(*conn).Read(0xc00007d730, {0xc00291e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001e38e48 sp=0xc001e38dc0 pc=0x8b176d +net.(*TCPConn).Read(0xc00007d730, {0xc00291e000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc001e38e90 sp=0xc001e38e48 pc=0x8d72f4 +github.com/nats-io/nats%2ego.(*natsReader).Read(0xc002853480) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1890 +0x1a9 fp=0xc001e38f10 sp=0xc001e38e90 pc=0x1f6d169 +github.com/nats-io/nats%2ego.(*Conn).readLoop(0xc00291a000) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2917 +0x1f8 fp=0xc001e38fc0 sp=0xc001e38f10 pc=0x1f75fb8 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x34 fp=0xc001e38fe0 sp=0xc001e38fc0 pc=0x1f71514 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e38fe8 sp=0xc001e38fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 886 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x44e + +goroutine 1103 [chan receive]: +runtime.gopark(0x4aef8c?, 0x4b0a55?, 0x28?, 0x91?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002347e18 sp=0xc002347df8 pc=0x4777ae +runtime.chanrecv(0xc002766de0, 0xc002347ed0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002347e90 sp=0xc002347e18 pc=0x441325 +runtime.chanrecv2(0xc001659128?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc002347eb8 sp=0xc002347e90 pc=0x440f92 +github.com/nats-io/nats%2ego.(*Conn).flusher(0xc001659100) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:3324 +0x1a5 fp=0xc002347fc0 sp=0xc002347eb8 pc=0x1f792e5 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x34 fp=0xc002347fe0 sp=0xc002347fc0 pc=0x1f714b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002347fe8 sp=0xc002347fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 1017 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x4cb + +goroutine 1043 [select]: +runtime.gopark(0xc002337f70?, 0x2?, 0xb4?, 0x1b?, 0xc002337ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002337d20 sp=0xc002337d00 pc=0x4777ae +runtime.selectgo(0xc002337f70, 0xc002337eb8, 0x472ddb?, 0x0, 0x4ad421?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002337e70 sp=0xc002337d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001fabac0, {0x5b426e0, 0xc002338eb0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc002337fa0 sp=0xc002337e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc002337fe0 sp=0xc002337fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002337fe8 sp=0xc002337fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1017 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1044 [select]: +runtime.gopark(0xc002335f70?, 0x2?, 0xb4?, 0x1b?, 0xc002335ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002335d20 sp=0xc002335d00 pc=0x4777ae +runtime.selectgo(0xc002335f70, 0xc002335eb8, 0x4b3745?, 0x0, 0xc001fefb48?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002335e70 sp=0xc002335d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001fabb20, {0x5b426e0, 0xc002338f00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc002335fa0 sp=0xc002335e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc002335fe0 sp=0xc002335fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002335fe8 sp=0xc002335fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1017 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1037 [select]: +runtime.gopark(0xc0029e1f70?, 0x2?, 0xb4?, 0x1b?, 0xc0029e1ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029e1d20 sp=0xc0029e1d00 pc=0x4777ae +runtime.selectgo(0xc0029e1f70, 0xc0029e1eb8, 0xc00213deb8?, 0x0, 0xc00213df10?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029e1e70 sp=0xc0029e1d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002489380, {0x5b426e0, 0xc002055130}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0029e1fa0 sp=0xc0029e1e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0029e1fe0 sp=0xc0029e1fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029e1fe8 sp=0xc0029e1fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 823 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 879 [IO wait]: +runtime.gopark(0xc002358fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002358f98 sp=0xc002358f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002358fd0 sp=0xc002358f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494af8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002358ff0 sp=0xc002358fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc002305a20, 0xc00251ac00?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002359038 sp=0xc002358ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc002305a00, {0xc00251ac00, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc002359128 sp=0xc002359038 pc=0x50e5a5 +net.(*netFD).Read(0xc002305a00, {0xc00251ac00, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc002359188 sp=0xc002359128 pc=0x89342b +net.(*conn).Read(0xc001d6fd18, {0xc00251ac00, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc002359210 sp=0xc002359188 pc=0x8b176d +net.(*UnixConn).Read(0xc001d6fd18, {0xc00251ac00, 0x600, 0x600}) + :1 +0x54 fp=0xc002359258 sp=0xc002359210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0032a2a08, {0xc00251ac00, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0023592b8 sp=0xc002359258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0017cb0a8, {0x5b0f6c0, 0xc0032a2a08}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc002359320 sp=0xc0023592b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc0017cae00, {0x5b0f840?, 0xc001d6fd18}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc002359390 sp=0xc002359320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc0017cae00, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0023599c0 sp=0xc002359390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc0017cae00, {0xc002526000, 0x8000, 0xc0030fc960?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc002359aa8 sp=0xc0023599c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001d87a40, {0xc002526000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc002359af0 sp=0xc002359aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc00251daa0, {0xc0011d5b60, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc002359bb0 sp=0xc002359af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00251daa0}, {0xc0011d5b60, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002359bf8 sp=0xc002359bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0011d5b60, 0x9, 0x9}, {0x5b0f0a0, 0xc00251daa0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc002359c98 sp=0xc002359bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011d5b20) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc002359dd0 sp=0xc002359c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001df1200, 0xc001852b40?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc002359fb0 sp=0xc002359dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc002359fe0 sp=0xc002359fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002359fe8 sp=0xc002359fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1045 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 1060 [select]: +runtime.gopark(0xc002470890?, 0x2?, 0x0?, 0x0?, 0xc00247083c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024706a8 sp=0xc002470688 pc=0x4777ae +runtime.selectgo(0xc002470890, 0xc002470838, 0x7f429a470b00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0024707f8 sp=0xc0024706a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0021e9d10, {0xc0024602e0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0024708c0 sp=0xc0024707f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0021e9d10, {0xc0024602e0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002470958 sp=0xc0024708c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00242fc50, {0xc0024602e0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0024709a0 sp=0xc002470958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc00242fc50}, {0xc0024602e0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0024709e8 sp=0xc0024709a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c96000, {0xc0024602e0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002470a58 sp=0xc0024709e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0024602d0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002470b28 sp=0xc002470a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc00112bc00?, 0xc001c96000, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002470c18 sp=0xc002470b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0024602d0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc00255dc20}, 0xc002470d70?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002470cd0 sp=0xc002470c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001b9cc30, {0x43dcf60?, 0xc00255dc20}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002470f08 sp=0xc002470cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002460420, {0x43dcf60, 0xc00255dc20}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002470ff0 sp=0xc002470f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002181440, {0x43dcf60, 0xc00255dc20}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002471028 sp=0xc002470ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc002447520) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc002471068 sp=0xc002471028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc002447520?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0024710a8 sp=0xc002471068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc002447530) + :1 +0x46 fp=0xc0024710e0 sp=0xc0024710a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0010e4630, {0x5b51368, 0xc002447530}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc002471320 sp=0xc0024710e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00158b250, {0x5b51038, 0xc002447520}) + :1 +0xaf fp=0xc002471370 sp=0xc002471320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00158b250}, {0x5b4a6f0?, 0xc002181440}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0024713b0 sp=0xc002471370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc002460420}, 0xc001e477d0?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002471400 sp=0xc0024713b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc002460420}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002471490 sp=0xc002471400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001b9cc30}, 0xc001e477d0, 0xc002453200) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0024717d0 sp=0xc002471490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001b9cc30}, 0x3f252a0?, 0xc002447450?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002471848 sp=0xc0024717d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc001c96000, 0xc0010e4810, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002471db0 sp=0xc002471848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc001c96000, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002471f28 sp=0xc002471db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002471fe0 sp=0xc002471f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002471fe8 sp=0xc002471fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1061 [select]: +runtime.gopark(0xc001fba890?, 0x2?, 0x98?, 0xa7?, 0xc001fba83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fba6a8 sp=0xc001fba688 pc=0x4777ae +runtime.selectgo(0xc001fba890, 0xc001fba838, 0x7f429a446500?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001fba7f8 sp=0xc001fba6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0021e9e00, {0xc00242ff90, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc001fba8c0 sp=0xc001fba7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0021e9e00, {0xc00242ff90, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001fba958 sp=0xc001fba8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00242fda0, {0xc00242ff90, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001fba9a0 sp=0xc001fba958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc00242fda0}, {0xc00242ff90, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001fba9e8 sp=0xc001fba9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c96120, {0xc00242ff90, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001fbaa58 sp=0xc001fba9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc00242ff80, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001fbab28 sp=0xc001fbaa58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc002453080?, 0xc001c96120, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001fbac18 sp=0xc001fbab28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc00242ff80, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc002462230}, 0xc0010e4480?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001fbacd0 sp=0xc001fbac18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001b9cb40, {0x43dcf60?, 0xc002462230}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001fbaf08 sp=0xc001fbacd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0024600f0, {0x43dcf60, 0xc002462230}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001fbaff0 sp=0xc001fbaf08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002181360, {0x43dcf60, 0xc002462230}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001fbb028 sp=0xc001fbaff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc002447320) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001fbb068 sp=0xc001fbb028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc002447320?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc001fbb0a8 sp=0xc001fbb068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc002447330) + :1 +0x46 fp=0xc001fbb0e0 sp=0xc001fbb0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0010e4630, {0x5b51368, 0xc002447330}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001fbb320 sp=0xc001fbb0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00158b250, {0x5b51038, 0xc002447320}) + :1 +0xaf fp=0xc001fbb370 sp=0xc001fbb320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00158b250}, {0x5b4a6f0?, 0xc002181360}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc001fbb3b0 sp=0xc001fbb370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc0024600f0}, 0xc001e475d8?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001fbb400 sp=0xc001fbb3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc0024600f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001fbb490 sp=0xc001fbb400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001b9cb40}, 0xc001e475d8, 0xc002452680) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001fbb7d0 sp=0xc001fbb490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001b9cb40}, 0x3f252a0?, 0xc002447250?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001fbb848 sp=0xc001fbb7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc001c96120, 0xc0010e4810, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001fbbdb0 sp=0xc001fbb848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc001c96120, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001fbbf28 sp=0xc001fbbdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001fbbfe0 sp=0xc001fbbf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fbbfe8 sp=0xc001fbbfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1351 [select]: +runtime.gopark(0xc0024cbf70?, 0x2?, 0xb4?, 0x1b?, 0xc0024cbebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024cbd20 sp=0xc0024cbd00 pc=0x4777ae +runtime.selectgo(0xc0024cbf70, 0xc0024cbeb8, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0024cbe70 sp=0xc0024cbd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0021800e0, {0x5b426e0, 0xc001abb400}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0024cbfa0 sp=0xc0024cbe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0024cbfe0 sp=0xc0024cbfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024cbfe8 sp=0xc0024cbfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1320 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 881 [select]: +runtime.gopark(0xc003035e48?, 0x2?, 0x0?, 0x0?, 0xc003035dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc003035c30 sp=0xc003035c10 pc=0x4777ae +runtime.selectgo(0xc003035e48, 0xc003035dc8, 0xc003035dd8?, 0x0, 0xc002256700?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc003035d80 sp=0xc003035c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001abbef0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc003035e78 sp=0xc003035d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0025208c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc003035f30 sp=0xc003035e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc003035fe0 sp=0xc003035f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc003035fe8 sp=0xc003035fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 878 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1074 [select]: +runtime.gopark(0xc001d44f70?, 0x4?, 0xc0?, 0x2?, 0xc001d44dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d44c00 sp=0xc001d44be0 pc=0x4777ae +runtime.selectgo(0xc001d44f70, 0xc001d44dd0, 0x40a16a0?, 0x0, 0xc0018553f0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001d44d50 sp=0xc001d44c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002230340) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc001d44fc0 sp=0xc001d44d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc001d44fe0 sp=0xc001d44fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d44fe8 sp=0xc001d44fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 878 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1075 [IO wait]: +runtime.gopark(0xc002236f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002236f40 sp=0xc002236f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002236f78 sp=0xc002236f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494908, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002236f98 sp=0xc002236f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001ab10a0, 0xc0015ddd40?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002236fe0 sp=0xc002236f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001ab1080, {0xc0015ddd40, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0022370d0 sp=0xc002236fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc001ab1080, {0xc0015ddd40, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc002237130 sp=0xc0022370d0 pc=0x89342b +net.(*conn).Read(0xc000b79fb8, {0xc0015ddd40, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0022371b8 sp=0xc002237130 pc=0x8b176d +net.(*UnixConn).Read(0xc000b79fb8, {0xc0015ddd40, 0x240, 0x240}) + :1 +0x54 fp=0xc002237200 sp=0xc0022371b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc00304bb78, {0xc0015ddd40, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc002237260 sp=0xc002237200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0022569a8, {0x5b0f6c0, 0xc00304bb78}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0022372c8 sp=0xc002237260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc002256700, {0x5b0f840?, 0xc000b79fb8}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc002237338 sp=0xc0022372c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc002256700, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc002237968 sp=0xc002237338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc002256700, {0xc002542000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc002237a50 sp=0xc002237968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001d87ac0, {0xc002542000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc002237a98 sp=0xc002237a50 pc=0x10969ed +bufio.(*Reader).Read(0xc00251dda0, {0xc0011d5c40, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc002237b58 sp=0xc002237a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00251dda0}, {0xc0011d5c40, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002237ba0 sp=0xc002237b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0011d5c40, 0x9, 0x9}, {0x5b0f0a0, 0xc00251dda0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc002237c40 sp=0xc002237ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011d5c00) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc002237d78 sp=0xc002237c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002230340, 0x78?, 0x1912368204258b60?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc002237ec8 sp=0xc002237d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc001852b40, {0x5b54e00?, 0xc002230340}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc002237f80 sp=0xc002237ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc002237fe0 sp=0xc002237f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002237fe8 sp=0xc002237fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 878 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1078 [select]: +runtime.gopark(0xc001d45fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001d45f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d45de8 sp=0xc001d45dc8 pc=0x4777ae +runtime.selectgo(0xc001d45fb0, 0xc001d45f70, 0xc00251de00?, 0x0, 0xc001d9dc80?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001d45f38 sp=0xc001d45de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001d45fe0 sp=0xc001d45f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d45fe8 sp=0xc001d45fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1058 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1079 [select]: +runtime.gopark(0xc001d46fb0?, 0x2?, 0xb0?, 0x6d?, 0xc001d46ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d46d58 sp=0xc001d46d38 pc=0x4777ae +runtime.selectgo(0xc001d46fb0, 0xc001d46ee0, 0xe?, 0x0, 0x8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001d46ea8 sp=0xc001d46d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001d46fe0 sp=0xc001d46ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d46fe8 sp=0xc001d46fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1058 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1080 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002349d60 sp=0xc002349d40 pc=0x4777ae +runtime.chanrecv(0xc001d9df80, 0xc002349e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002349dd8 sp=0xc002349d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002349e00 sp=0xc002349dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002349fe0 sp=0xc002349e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002349fe8 sp=0xc002349fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1058 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1090 [chan receive]: +runtime.gopark(0x5b426a8?, 0x0?, 0xc8?, 0x7e?, 0x111478c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002401f08 sp=0xc002401ee8 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002401f80 sp=0xc002401f08 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0xc001d6fd18?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002401fa8 sp=0xc002401f80 pc=0x440f72 +github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore.func3() + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:99 +0x4a fp=0xc002401fe0 sp=0xc002401fa8 pc=0x24a06aa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002401fe8 sp=0xc002401fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore in goroutine 823 + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:98 +0x4cb + +goroutine 1091 [select]: +runtime.gopark(0xc00235db10?, 0x13b0513?, 0x0?, 0x1?, 0xc002402ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002402d30 sp=0xc002402d10 pc=0x4777ae +runtime.selectgo(0xc002402f60, 0xc002402ec8, 0x4ad421?, 0x1, 0x12ad646?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002402e80 sp=0xc002402d30 pc=0x48916b +github.com/lestrrat-go/backoff/v2.(*controller).loop(0xc002341da0) + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:64 +0x318 fp=0xc002402fc0 sp=0xc002402e80 pc=0x13b0538 +github.com/lestrrat-go/backoff/v2.newController.func1() + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x34 fp=0xc002402fe0 sp=0xc002402fc0 pc=0x13b01f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002402fe8 sp=0xc002402fe0 pc=0x4ad421 +created by github.com/lestrrat-go/backoff/v2.newController in goroutine 823 + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x4dd + +goroutine 1092 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc002345930?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00235bdd8 sp=0xc00235bdb8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc002722990, 0xb) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc00235be28 sp=0xc00235bdd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc002722980) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc00235be58 sp=0xc00235be28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc001658a00, 0xc002345930) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc00235bfb0 sp=0xc00235be58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc00235bfe0 sp=0xc00235bfb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00235bfe8 sp=0xc00235bfe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 823 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 1104 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc00237d5a0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002346f08 sp=0xc002346ee8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc002a20410, 0x0) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc002346f58 sp=0xc002346f08 pc=0x4a9339 +sync.(*Cond).Wait(0xc002a20400) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc002346f88 sp=0xc002346f58 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*asyncCallbacksHandler).asyncCBDispatcher(0xc00237d5a0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2842 +0x153 fp=0xc002346fc0 sp=0xc002346f88 pc=0x1f759b3 +github.com/nats-io/nats%2ego.Options.Connect.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x34 fp=0xc002346fe0 sp=0xc002346fc0 pc=0x1f6a534 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002346fe8 sp=0xc002346fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.Options.Connect in goroutine 1017 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x86b + +goroutine 1105 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc002a1b5f0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002355dd8 sp=0xc002355db8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc002a20610, 0x1f) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc002355e28 sp=0xc002355dd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc002a20600) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc002355e58 sp=0xc002355e28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc001659100, 0xc002a1b5f0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc002355fb0 sp=0xc002355e58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc002355fe0 sp=0xc002355fb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002355fe8 sp=0xc002355fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 1017 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 1107 [chan receive]: +runtime.gopark(0xc00274ae40?, 0x4be4a5?, 0x28?, 0xa0?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00274ae18 sp=0xc00274adf8 pc=0x4777ae +runtime.chanrecv(0xc002767020, 0xc00274aed0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00274ae90 sp=0xc00274ae18 pc=0x441325 +runtime.chanrecv2(0xc00291a028?, 0x4aead9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc00274aeb8 sp=0xc00274ae90 pc=0x440f92 +github.com/nats-io/nats%2ego.(*Conn).flusher(0xc00291a000) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:3324 +0x1a5 fp=0xc00274afc0 sp=0xc00274aeb8 pc=0x1f792e5 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x34 fp=0xc00274afe0 sp=0xc00274afc0 pc=0x1f714b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00274afe8 sp=0xc00274afe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 886 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x4cb + +goroutine 1108 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc0024890a0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002348f08 sp=0xc002348ee8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc002853450, 0x0) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc002348f58 sp=0xc002348f08 pc=0x4a9339 +sync.(*Cond).Wait(0xc002853440) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc002348f88 sp=0xc002348f58 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*asyncCallbacksHandler).asyncCBDispatcher(0xc0024890a0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2842 +0x153 fp=0xc002348fc0 sp=0xc002348f88 pc=0x1f759b3 +github.com/nats-io/nats%2ego.Options.Connect.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x34 fp=0xc002348fe0 sp=0xc002348fc0 pc=0x1f6a534 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002348fe8 sp=0xc002348fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.Options.Connect in goroutine 886 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x86b + +goroutine 1109 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002749f08 sp=0xc002749ee8 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002749f80 sp=0xc002749f08 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0xc001e2fa60?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002749fa8 sp=0xc002749f80 pc=0x440f72 +github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore.func3() + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:99 +0x4a fp=0xc002749fe0 sp=0xc002749fa8 pc=0x24a06aa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002749fe8 sp=0xc002749fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore in goroutine 886 + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:98 +0x4cb + +goroutine 1110 [select]: +runtime.gopark(0xc002767270?, 0x13b0513?, 0x0?, 0x1?, 0xc00274decc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00274dd30 sp=0xc00274dd10 pc=0x4777ae +runtime.selectgo(0xc00274df60, 0xc00274dec8, 0xc00106e748?, 0x1, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00274de80 sp=0xc00274dd30 pc=0x48916b +github.com/lestrrat-go/backoff/v2.(*controller).loop(0xc002742c60) + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:64 +0x318 fp=0xc00274dfc0 sp=0xc00274de80 pc=0x13b0538 +github.com/lestrrat-go/backoff/v2.newController.func1() + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x34 fp=0xc00274dfe0 sp=0xc00274dfc0 pc=0x13b01f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00274dfe8 sp=0xc00274dfe0 pc=0x4ad421 +created by github.com/lestrrat-go/backoff/v2.newController in goroutine 886 + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x4dd + +goroutine 1111 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc002a1b930?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002357dd8 sp=0xc002357db8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc002a20990, 0xa) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc002357e28 sp=0xc002357dd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc002a20980) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc002357e58 sp=0xc002357e28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc00291a000, 0xc002a1b930) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc002357fb0 sp=0xc002357e58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc002357fe0 sp=0xc002357fb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002357fe8 sp=0xc002357fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 886 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 1133 [chan receive]: +runtime.gopark(0x0?, 0xc002c01e10?, 0xa2?, 0x16?, 0xc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00212b318 sp=0xc00212b2f8 pc=0x4777ae +runtime.chanrecv(0xc0027e5c20, 0xc00212b478, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00212b390 sp=0xc00212b318 pc=0x441325 +runtime.chanrecv2(0xc000b6f040?, 0x5b426a8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc00212b3b8 sp=0xc00212b390 pc=0x440f92 +github.com/rancher/opni/pkg/management.(*Server).WatchClusters(0xc0010c0e70, 0xc002c01e30, {0x5b4cb30, 0xc000b6f040}) + /home/ec2-user/opni/pkg/management/clusters.go:93 +0x26d fp=0xc00212b4a8 sp=0xc00212b3b8 pc=0x230a3cd +github.com/rancher/opni/pkg/apis/management/v1._Management_WatchClusters_Handler({0x4537040?, 0xc0010c0e70}, {0x5b4ab30, 0xc002c01e00}) + /home/ec2-user/opni/pkg/apis/management/v1/management_grpc.pb.go:736 +0xfa fp=0xc00212b508 sp=0xc00212b4a8 pc=0x178093a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4537040, 0xc0010c0e70}, {0x5b492f8, 0xc000414960}, 0xc0028fce88, 0x47ec6c0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc00212b848 sp=0xc00212b508 pc=0x12b0e02 +google.golang.org/grpc.(*Server).processStreamingRPC(0xc001ebfc20, {0x5b54e00, 0xc0021d44e0}, 0xc0003bfd40, 0xc001be2420, 0x7716c60, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc00212bdb0 sp=0xc00212b848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc001ebfc20, {0x5b54e00, 0xc0021d44e0}, 0xc0003bfd40, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc00212bf28 sp=0xc00212bdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc00212bfe0 sp=0xc00212bf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00212bfe8 sp=0xc00212bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1007 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1143 [select]: +runtime.gopark(0xc002189ed0?, 0x2?, 0xb4?, 0x1b?, 0xc002189e4c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002189c90 sp=0xc002189c70 pc=0x4777ae +runtime.selectgo(0xc002189ed0, 0xc002189e48, 0x0?, 0x0, 0x406e00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002189de0 sp=0xc002189c90 pc=0x48916b +github.com/rancher/opni/pkg/management.(*Server).listenAndServeGrpc(0xc0010c0e70, {0x5b426e0, 0xc0012c2730}) + /home/ec2-user/opni/pkg/management/server.go:215 +0x3cd fp=0xc002189f20 sp=0xc002189de0 pc=0x231e52d +github.com/rancher/opni/pkg/management.(*Server).ListenAndServe.func1() + /home/ec2-user/opni/pkg/management/server.go:181 +0x4b fp=0xc002189fa0 sp=0xc002189f20 pc=0x231e0ab +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc002189fe0 sp=0xc002189fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002189fe8 sp=0xc002189fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1084 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1233 [select]: +runtime.gopark(0xc002127f10?, 0x3?, 0xb4?, 0x1b?, 0xc002127e92?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002127ca8 sp=0xc002127c88 pc=0x4777ae +runtime.selectgo(0xc002127f10, 0xc002127e8c, 0x5b126c0?, 0x0, 0x1?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002127df8 sp=0xc002127ca8 pc=0x48916b +google.golang.org/grpc.(*addrConn).resetTransport(0xc001f13200) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/clientconn.go:1368 +0x3ef fp=0xc002127f50 sp=0xc002127df8 pc=0x116334f +google.golang.org/grpc.(*addrConn).connect(0xc001f13200) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/clientconn.go:993 +0x23b fp=0xc002127fc0 sp=0xc002127f50 pc=0x116033b +google.golang.org/grpc.(*acBalancerWrapper).Connect.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/balancer_conn_wrappers.go:398 +0x34 fp=0xc002127fe0 sp=0xc002127fc0 pc=0x1157434 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002127fe8 sp=0xc002127fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*acBalancerWrapper).Connect in goroutine 1040 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/balancer_conn_wrappers.go:398 +0xa7 + +goroutine 1135 [select]: +runtime.gopark(0xc002407fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002407f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002407de8 sp=0xc002407dc8 pc=0x4777ae +runtime.selectgo(0xc002407fb0, 0xc002407f70, 0x1000000?, 0x0, 0x116e9f5?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002407f38 sp=0xc002407de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002407fe0 sp=0xc002407f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002407fe8 sp=0xc002407fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1139 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1124 [chan receive]: +runtime.gopark(0x2106cfa?, 0x1177efa?, 0x67?, 0x6b?, 0xc001852b40?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002059f00 sp=0xc002059ee0 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002059f78 sp=0xc002059f00 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0xc001d4d6a0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002059fa0 sp=0xc002059f78 pc=0x440f72 +github.com/rancher/opni/plugins/metrics/pkg/cortex.NewClientSet.func1() + /home/ec2-user/opni/plugins/metrics/pkg/cortex/clientset.go:555 +0x53 fp=0xc002059fe0 sp=0xc002059fa0 pc=0x3d9cfb3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002059fe8 sp=0xc002059fe0 pc=0x4ad421 +created by github.com/rancher/opni/plugins/metrics/pkg/cortex.NewClientSet in goroutine 823 + /home/ec2-user/opni/plugins/metrics/pkg/cortex/clientset.go:554 +0x999 + +goroutine 1144 [select]: +runtime.gopark(0xc001755ed0?, 0x2?, 0xb4?, 0x1b?, 0xc001755e1c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001755c68 sp=0xc001755c48 pc=0x4777ae +runtime.selectgo(0xc001755ed0, 0xc001755e18, 0x1?, 0x0, 0xc0027d4600?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001755db8 sp=0xc001755c68 pc=0x48916b +github.com/rancher/opni/pkg/management.(*Server).listenAndServeHttp(0xc0010c0e70, {0x5b426e0?, 0xc0012c2730}) + /home/ec2-user/opni/pkg/management/server.go:248 +0x625 fp=0xc001755f20 sp=0xc001755db8 pc=0x231ed05 +github.com/rancher/opni/pkg/management.(*Server).ListenAndServe.func2() + /home/ec2-user/opni/pkg/management/server.go:189 +0x4b fp=0xc001755fa0 sp=0xc001755f20 pc=0x231dfab +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc001755fe0 sp=0xc001755fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001755fe8 sp=0xc001755fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1084 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1087 [select]: +runtime.gopark(0xc002008300?, 0x3?, 0xa0?, 0xea?, 0xc001575caa?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00246d7d0 sp=0xc00246d7b0 pc=0x4777ae +runtime.selectgo(0xc002008300, 0xc001575ca4, 0x0?, 0x0, 0xc001bf0240?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00246d920 sp=0xc00246d7d0 pc=0x48916b +reflect.rselect({0xc00246db08, 0x3, 0x452a85?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:589 +0x345 fp=0xc00246d9d0 sp=0xc00246d920 pc=0x4a8f05 +reflect.Select({0xc000b9a000, 0x3, 0x2?}) + /home/ec2-user/sdk/go1.21.0/src/reflect/value.go:3104 +0xb8a fp=0xc00246db98 sp=0xc00246d9d0 pc=0x57c44a +github.com/rancher/opni/pkg/util.WaitAll({0x5b426e0, 0xc00139cf00}, 0xc001bf0200, {0xc00246de08, 0x2, 0x2}) + /home/ec2-user/opni/pkg/util/servers.go:51 +0x3ea fp=0xc00246dd18 sp=0xc00246db98 pc=0x176ae8a +github.com/rancher/opni/pkg/gateway.(*GatewayHTTPServer).ListenAndServe(0xc0013042a0, {0x5b426e0, 0xc0007639a0}) + /home/ec2-user/opni/pkg/gateway/http.go:185 +0x5a7 fp=0xc00246de68 sp=0xc00246dd18 pc=0x22b8427 +github.com/rancher/opni/pkg/gateway.(*Gateway).ListenAndServe.func1() + /home/ec2-user/opni/pkg/gateway/gateway.go:327 +0x86 fp=0xc00246dfa0 sp=0xc00246de68 pc=0x22b28a6 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc00246dfe0 sp=0xc00246dfa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00246dfe8 sp=0xc00246dfe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1085 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1088 [select]: +runtime.gopark(0xc001bebe38?, 0x2?, 0xb4?, 0x1b?, 0xc001bebc5c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001beba58 sp=0xc001beba38 pc=0x4777ae +runtime.selectgo(0xc001bebe38, 0xc001bebc58, 0x0?, 0x0, 0x4113100?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001bebba8 sp=0xc001beba58 pc=0x48916b +github.com/rancher/opni/pkg/gateway.(*GatewayGRPCServer).ListenAndServe(0xc001df9b00, {0x5b426e0, 0xc0007639a0}) + /home/ec2-user/opni/pkg/gateway/grpc.go:107 +0x106f fp=0xc001bebe68 sp=0xc001bebba8 pc=0x22b4b0f +github.com/rancher/opni/pkg/gateway.(*Gateway).ListenAndServe.func2() + /home/ec2-user/opni/pkg/gateway/gateway.go:338 +0x86 fp=0xc001bebfa0 sp=0xc001bebe68 pc=0x22b2686 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc001bebfe0 sp=0xc001bebfa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bebfe8 sp=0xc001bebfe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1085 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1173 [chan receive]: +runtime.gopark(0xc001441b00?, 0x0?, 0x0?, 0x4e?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002755ee8 sp=0xc002755ec8 pc=0x4777ae +runtime.chanrecv(0xc0021fe9c0, 0xc002755fa0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002755f60 sp=0xc002755ee8 pc=0x441325 +runtime.chanrecv2(0x0?, 0xc000b31560?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc002755f88 sp=0xc002755f60 pc=0x440f92 +google.golang.org/grpc.(*Server).serverWorker(0xc00170e000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:593 +0x85 fp=0xc002755fc0 sp=0xc002755f88 pc=0x11744a5 +google.golang.org/grpc.(*Server).initServerWorkers.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0x34 fp=0xc002755fe0 sp=0xc002755fc0 pc=0x1174994 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002755fe8 sp=0xc002755fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).initServerWorkers in goroutine 1088 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0xa5 + +goroutine 1175 [chan receive]: +runtime.gopark(0xc0018f5e60?, 0x0?, 0x0?, 0x4e?, 0xc001000000?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0027ddee8 sp=0xc0027ddec8 pc=0x4777ae +runtime.chanrecv(0xc0021fe9c0, 0xc0027ddfa0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0027ddf60 sp=0xc0027ddee8 pc=0x441325 +runtime.chanrecv2(0x1?, 0xc002488880?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0027ddf88 sp=0xc0027ddf60 pc=0x440f92 +google.golang.org/grpc.(*Server).serverWorker(0xc00170e000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:593 +0x85 fp=0xc0027ddfc0 sp=0xc0027ddf88 pc=0x11744a5 +google.golang.org/grpc.(*Server).initServerWorkers.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0x34 fp=0xc0027ddfe0 sp=0xc0027ddfc0 pc=0x1174994 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0027ddfe8 sp=0xc0027ddfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).initServerWorkers in goroutine 1088 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0xa5 + +goroutine 1145 [select]: +runtime.gopark(0xc0017c9f70?, 0x2?, 0xb4?, 0x1b?, 0xc0017c9ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017c9d20 sp=0xc0017c9d00 pc=0x4777ae +runtime.selectgo(0xc0017c9f70, 0xc0017c9eb8, 0x4b3745?, 0x0, 0xc00106e748?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017c9e70 sp=0xc0017c9d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0027849a0, {0x5b426e0, 0xc0012c29b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0017c9fa0 sp=0xc0017c9e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0017c9fe0 sp=0xc0017c9fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017c9fe8 sp=0xc0017c9fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1144 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1146 [select]: +runtime.gopark(0xc0029d9f70?, 0x2?, 0xb4?, 0x1b?, 0xc0029d9ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029d9d20 sp=0xc0029d9d00 pc=0x4777ae +runtime.selectgo(0xc0029d9f70, 0xc0029d9eb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029d9e70 sp=0xc0029d9d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002784a00, {0x5b426e0, 0xc0012c2a00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0029d9fa0 sp=0xc0029d9e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0029d9fe0 sp=0xc0029d9fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029d9fe8 sp=0xc0029d9fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1144 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1149 [select]: +runtime.gopark(0xc0017c3f70?, 0x2?, 0xb4?, 0x1b?, 0xc0017c3ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017c3d20 sp=0xc0017c3d00 pc=0x4777ae +runtime.selectgo(0xc0017c3f70, 0xc0017c3eb8, 0xc001b70600?, 0x0, 0x440100?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017c3e70 sp=0xc0017c3d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0027854c0, {0x5b426e0, 0xc000b68870}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0017c3fa0 sp=0xc0017c3e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0017c3fe0 sp=0xc0017c3fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017c3fe8 sp=0xc0017c3fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 609 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1176 [chan receive]: +runtime.gopark(0xc001d8a7e0?, 0x0?, 0x0?, 0x4e?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0027e1ee8 sp=0xc0027e1ec8 pc=0x4777ae +runtime.chanrecv(0xc0021fe9c0, 0xc0027e1fa0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0027e1f60 sp=0xc0027e1ee8 pc=0x441325 +runtime.chanrecv2(0xc0000fd360?, 0xc002bf88e0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0027e1f88 sp=0xc0027e1f60 pc=0x440f92 +google.golang.org/grpc.(*Server).serverWorker(0xc00170e000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:593 +0x85 fp=0xc0027e1fc0 sp=0xc0027e1f88 pc=0x11744a5 +google.golang.org/grpc.(*Server).initServerWorkers.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0x34 fp=0xc0027e1fe0 sp=0xc0027e1fc0 pc=0x1174994 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0027e1fe8 sp=0xc0027e1fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).initServerWorkers in goroutine 1088 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0xa5 + +goroutine 1160 [select]: +runtime.gopark(0xc001980ed8?, 0x2?, 0xb4?, 0x1b?, 0xc001980e84?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001980cf8 sp=0xc001980cd8 pc=0x4777ae +runtime.selectgo(0xc001980ed8, 0xc001980e80, 0xc001980ea8?, 0x0, 0xc001980e90?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001980e48 sp=0xc001980cf8 pc=0x48916b +github.com/rancher/opni/pkg/util.ServeHandler({0x5b426e0, 0xc00139cf00}, {0x5b13c00?, 0xc0012c4b60}, {0x5b2a6d0?, 0xc00296ff00}) + /home/ec2-user/opni/pkg/util/servers.go:21 +0x26a fp=0xc001980f08 sp=0xc001980e48 pc=0x176a94a +github.com/rancher/opni/pkg/gateway.(*GatewayHTTPServer).ListenAndServe.func2() + /home/ec2-user/opni/pkg/gateway/http.go:182 +0x1b9 fp=0xc001980fa0 sp=0xc001980f08 pc=0x22b8699 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc001980fe0 sp=0xc001980fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001980fe8 sp=0xc001980fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1087 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1177 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029359e0 sp=0xc0029359c0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002935a18 sp=0xc0029359e0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163adc0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002935a38 sp=0xc002935a18 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0016f8a20, 0x4b0501?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002935a80 sp=0xc002935a38 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0016f8a00) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc002935b70 sp=0xc002935a80 pc=0x513f65 +net.(*netFD).accept(0xc0016f8a00) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc002935c68 sp=0xc002935b70 pc=0x8967be +net.(*TCPListener).accept(0xc0016353c0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc002935cb0 sp=0xc002935c68 pc=0x8c383e +net.(*TCPListener).Accept(0xc0016353c0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc002935d20 sp=0xc002935cb0 pc=0x8c1ba5 +google.golang.org/grpc.(*Server).Serve(0xc00170e000, {0x5b2a6d0, 0xc0016353c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc002935f50 sp=0xc002935d20 pc=0x117789e +github.com/rancher/opni/pkg/gateway.(*GatewayGRPCServer).ListenAndServe.func1() + /home/ec2-user/opni/pkg/gateway/grpc.go:105 +0x47 fp=0xc002935fa0 sp=0xc002935f50 pc=0x22b4c27 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc002935fe0 sp=0xc002935fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002935fe8 sp=0xc002935fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1088 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1161 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00215c9e0 sp=0xc00215c9c0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00215ca18 sp=0xc00215c9e0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163a6f8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00215ca38 sp=0xc00215ca18 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00182c120, 0x4b0501?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00215ca80 sp=0xc00215ca38 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc00182c100) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc00215cb70 sp=0xc00215ca80 pc=0x513f65 +net.(*netFD).accept(0xc00182c100) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc00215cc68 sp=0xc00215cb70 pc=0x8967be +net.(*TCPListener).accept(0xc00296ff00) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc00215ccb0 sp=0xc00215cc68 pc=0x8c383e +net.(*TCPListener).Accept(0xc00296ff00) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc00215cd20 sp=0xc00215ccb0 pc=0x8c1ba5 +net/http.(*onceCloseListener).Accept(0xc002008330) + :1 +0x44 fp=0xc00215cd58 sp=0xc00215cd20 pc=0xab5544 +net/http.(*Server).Serve(0xc000b9c000, {0x5b2a6d0, 0xc00296ff00}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc00215cf50 sp=0xc00215cd58 pc=0xa7859f +github.com/rancher/opni/pkg/util.ServeHandler.func1() + /home/ec2-user/opni/pkg/util/servers.go:19 +0x47 fp=0xc00215cfa0 sp=0xc00215cf50 pc=0x176aa67 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc00215cfe0 sp=0xc00215cfa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00215cfe8 sp=0xc00215cfe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1160 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1178 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001823988 sp=0xc001823968 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0018239c0 sp=0xc001823988 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163abd0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0018239e0 sp=0xc0018239c0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00182c0a0, 0xc001738301?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001823a28 sp=0xc0018239e0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc00182c080) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001823b18 sp=0xc001823a28 pc=0x513f65 +net.(*netFD).accept(0xc00182c080) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001823c10 sp=0xc001823b18 pc=0x8967be +net.(*TCPListener).accept(0xc00296fea0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc001823c58 sp=0xc001823c10 pc=0x8c383e +net.(*TCPListener).Accept(0xc00296fea0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc001823cc8 sp=0xc001823c58 pc=0x8c1ba5 +crypto/tls.(*listener).Accept(0xc0028fd4d0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/tls.go:66 +0x43 fp=0xc001823d20 sp=0xc001823cc8 pc=0x9b2d43 +net/http.(*onceCloseListener).Accept(0xc002172ed0) + :1 +0x44 fp=0xc001823d58 sp=0xc001823d20 pc=0xab5544 +net/http.(*Server).Serve(0xc0016702d0, {0x5b2a5b0, 0xc0028fd4d0}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc001823f50 sp=0xc001823d58 pc=0xa7859f +github.com/rancher/opni/pkg/util.ServeHandler.func1() + /home/ec2-user/opni/pkg/util/servers.go:19 +0x47 fp=0xc001823fa0 sp=0xc001823f50 pc=0x176aa67 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc001823fe0 sp=0xc001823fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001823fe8 sp=0xc001823fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1159 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1317 [syscall]: +syscall.Syscall6(0x100000045?, 0x47efdc8?, 0xc0021fceb0?, 0xc0020099e0?, 0xc00099bdc8?, 0xc00099bdd8?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/syscall/syscall_linux.go:91 +0x3d fp=0xc002748d38 sp=0xc002748cb0 pc=0x4d1b1d +os.(*Process).blockUntilWaitable(0xc0009537d0) + /home/ec2-user/sdk/go1.21.0/src/os/wait_waitid.go:32 +0x9f fp=0xc002748e18 sp=0xc002748d38 pc=0x529c5f +os.(*Process).wait(0xc0009537d0) + /home/ec2-user/sdk/go1.21.0/src/os/exec_unix.go:22 +0x4b fp=0xc002748e98 sp=0xc002748e18 pc=0x51e0ab +os.(*Process).Wait(...) + /home/ec2-user/sdk/go1.21.0/src/os/exec.go:134 +os/exec.(*Cmd).Wait(0xc0026c3340) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:890 +0xb9 fp=0xc002748f60 sp=0xc002748e98 pc=0x657279 +github.com/onsi/gomega/gexec.(*Session).monitorForExit(0xc002460840, 0x0?) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:196 +0x3f fp=0xc002748fb0 sp=0xc002748f60 pc=0x233731f +github.com/onsi/gomega/gexec.Start.func1() + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:96 +0x45 fp=0xc002748fe0 sp=0xc002748fb0 pc=0x2336d65 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002748fe8 sp=0xc002748fe0 pc=0x4ad421 +created by github.com/onsi/gomega/gexec.Start in goroutine 1250 + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:96 +0x9eb + +goroutine 1163 [select]: +runtime.gopark(0xc00190be48?, 0x2?, 0x0?, 0x0?, 0xc00190bdcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00190bc30 sp=0xc00190bc10 pc=0x4777ae +runtime.selectgo(0xc00190be48, 0xc00190bdc8, 0xc00190bdd8?, 0x0, 0xc000641580?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00190bd80 sp=0xc00190bc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00139d270, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00190be78 sp=0xc00190bd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0018b8770) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00190bf30 sp=0xc00190be78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc00190bfe0 sp=0xc00190bf30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00190bfe8 sp=0xc00190bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1162 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1164 [select]: +runtime.gopark(0xc00181ff70?, 0x4?, 0x58?, 0xfc?, 0xc00181fdd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00181fc00 sp=0xc00181fbe0 pc=0x4777ae +runtime.selectgo(0xc00181ff70, 0xc00181fdd0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00181fd50 sp=0xc00181fc00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0021d5520) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc00181ffc0 sp=0xc00181fd50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc00181ffe0 sp=0xc00181ffc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00181ffe8 sp=0xc00181ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1162 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1165 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00215d7d8 sp=0xc00215d7b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00215d810 sp=0xc00215d7d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494430, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00215d830 sp=0xc00215d810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00182c1a0, 0xc001880000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00215d878 sp=0xc00215d830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00182c180, {0xc001880000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00215d968 sp=0xc00215d878 pc=0x50e5a5 +net.(*netFD).Read(0xc00182c180, {0xc001880000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00215d9c8 sp=0xc00215d968 pc=0x89342b +net.(*conn).Read(0xc000641580, {0xc001880000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00215da50 sp=0xc00215d9c8 pc=0x8b176d +net.(*TCPConn).Read(0xc000641580, {0xc001880000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc00215da98 sp=0xc00215da50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc000c4e540, {0xc000b9a120, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00215db58 sp=0xc00215da98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000c4e540}, {0xc000b9a120, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00215dba0 sp=0xc00215db58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000b9a120, 0x9, 0x9}, {0x5b0f0a0, 0xc000c4e540}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00215dc40 sp=0xc00215dba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000b9a0e0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00215dd78 sp=0xc00215dc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0021d5520, 0x28?, 0x5b426a8?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc00215dec8 sp=0xc00215dd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c1e0, {0x5b54e00?, 0xc0021d5520}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc00215df80 sp=0xc00215dec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc00215dfe0 sp=0xc00215df80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00215dfe8 sp=0xc00215dfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1162 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1212 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0022db830 sp=0xc0022db810 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0022db868 sp=0xc0022db830 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243493f58, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0022db888 sp=0xc0022db868 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0016f8d20, 0xc0022c0000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0022db8d0 sp=0xc0022db888 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0016f8d00, {0xc0022c0000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0022db9c0 sp=0xc0022db8d0 pc=0x50e5a5 +net.(*netFD).Read(0xc0016f8d00, {0xc0022c0000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0022dba20 sp=0xc0022db9c0 pc=0x89342b +net.(*conn).Read(0xc001d6e9f8, {0xc0022c0000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0022dbaa8 sp=0xc0022dba20 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6e9f8, {0xc0022c0000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc0022dbaf0 sp=0xc0022dbaa8 pc=0x8d72f4 +bufio.(*Reader).Read(0xc00150d9e0, {0xc0017031c0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0022dbbb0 sp=0xc0022dbaf0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00150d9e0}, {0xc0017031c0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0022dbbf8 sp=0xc0022dbbb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0017031c0, 0x9, 0x9}, {0x5b0f0a0, 0xc00150d9e0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0022dbc98 sp=0xc0022dbbf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001703180) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0022dbdd0 sp=0xc0022dbc98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000eedb00, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0022dbfb0 sp=0xc0022dbdd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0022dbfe0 sp=0xc0022dbfb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0022dbfe8 sp=0xc0022dbfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1151 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 1183 [IO wait]: +runtime.gopark(0x448485?, 0x431af0?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00293b708 sp=0xc00293b6e8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00293b740 sp=0xc00293b708 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494338, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00293b760 sp=0xc00293b740 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0016f8b20, 0xc001847000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00293b7a8 sp=0xc00293b760 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0016f8b00, {0xc001847000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00293b898 sp=0xc00293b7a8 pc=0x50e5a5 +net.(*netFD).Read(0xc0016f8b00, {0xc001847000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00293b8f8 sp=0xc00293b898 pc=0x89342b +net.(*conn).Read(0xc001d6e968, {0xc001847000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00293b980 sp=0xc00293b8f8 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6e968, {0xc001847000, 0x1000, 0x1000}) + :1 +0x54 fp=0xc00293b9c8 sp=0xc00293b980 pc=0x8d72f4 +net/http.(*persistConn).Read(0xc0012c17a0, {0xc001847000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1954 +0xf9 fp=0xc00293ba48 sp=0xc00293b9c8 pc=0xa9c1d9 +bufio.(*Reader).fill(0xc00150d5c0) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:113 +0x29a fp=0xc00293bad8 sp=0xc00293ba48 pc=0x5f863a +bufio.(*Reader).Peek(0xc00150d5c0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:151 +0xc7 fp=0xc00293bb60 sp=0xc00293bad8 pc=0x5f88e7 +net/http.(*persistConn).readLoop(0xc0012c17a0) + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:2118 +0x2db fp=0xc00293bfc0 sp=0xc00293bb60 pc=0xa9d41b +net/http.(*Transport).dialConn.func5() + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1776 +0x34 fp=0xc00293bfe0 sp=0xc00293bfc0 pc=0xa9aef4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00293bfe8 sp=0xc00293bfe0 pc=0x4ad421 +created by net/http.(*Transport).dialConn in goroutine 1179 + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1776 +0x24da + +goroutine 1184 [select]: +runtime.gopark(0xc002746f90?, 0x2?, 0x10?, 0x6d?, 0xc002746e6c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002746cc8 sp=0xc002746ca8 pc=0x4777ae +runtime.selectgo(0xc002746f90, 0xc002746e68, 0xc0011cae80?, 0x0, 0xc0021732c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002746e18 sp=0xc002746cc8 pc=0x48916b +net/http.(*persistConn).writeLoop(0xc0012c17a0) + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:2421 +0x196 fp=0xc002746fc0 sp=0xc002746e18 pc=0xa9fe36 +net/http.(*Transport).dialConn.func6() + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1777 +0x34 fp=0xc002746fe0 sp=0xc002746fc0 pc=0xa9ae94 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002746fe8 sp=0xc002746fe0 pc=0x4ad421 +created by net/http.(*Transport).dialConn in goroutine 1179 + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1777 +0x256b + +goroutine 1206 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017c5830 sp=0xc0017c5810 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0017c5868 sp=0xc0017c5830 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494148, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0017c5888 sp=0xc0017c5868 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0016f8ba0, 0xc0021de000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0017c58d0 sp=0xc0017c5888 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0016f8b80, {0xc0021de000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0017c59c0 sp=0xc0017c58d0 pc=0x50e5a5 +net.(*netFD).Read(0xc0016f8b80, {0xc0021de000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0017c5a20 sp=0xc0017c59c0 pc=0x89342b +net.(*conn).Read(0xc001d6e988, {0xc0021de000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0017c5aa8 sp=0xc0017c5a20 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6e988, {0xc0021de000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc0017c5af0 sp=0xc0017c5aa8 pc=0x8d72f4 +bufio.(*Reader).Read(0xc00150d740, {0xc001703000, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0017c5bb0 sp=0xc0017c5af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00150d740}, {0xc001703000, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0017c5bf8 sp=0xc0017c5bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc001703000, 0x9, 0x9}, {0x5b0f0a0, 0xc00150d740}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0017c5c98 sp=0xc0017c5bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001702fc0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0017c5dd0 sp=0xc0017c5c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000eed680, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0017c5fb0 sp=0xc0017c5dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0017c5fe0 sp=0xc0017c5fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017c5fe8 sp=0xc0017c5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1185 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 1150 [select]: +runtime.gopark(0xc001f3ff70?, 0x2?, 0xb4?, 0x1b?, 0xc001f3febc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f3fd20 sp=0xc001f3fd00 pc=0x4777ae +runtime.selectgo(0xc001f3ff70, 0xc001f3feb8, 0x4b3745?, 0x0, 0xc00106e748?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001f3fe70 sp=0xc001f3fd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002785520, {0x5b426e0, 0xc000b688c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001f3ffa0 sp=0xc001f3fe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001f3ffe0 sp=0xc001f3ffa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f3ffe8 sp=0xc001f3ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 609 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1207 [select]: +runtime.gopark(0xc0021f3e70?, 0x2?, 0x0?, 0x0?, 0xc0021f3df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0021f3c58 sp=0xc0021f3c38 pc=0x4777ae +runtime.selectgo(0xc0021f3e70, 0xc0021f3df0, 0xc0021f3e00?, 0x0, 0xc001d6e988?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0021f3da8 sp=0xc0021f3c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00099bdb0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0021f3ea0 sp=0xc0021f3da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001704c40) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0021f3f58 sp=0xc0021f3ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc0021f3fe0 sp=0xc0021f3f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0021f3fe8 sp=0xc0021f3fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1185 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1214 [select]: +runtime.gopark(0xc00232fe48?, 0x2?, 0x0?, 0x0?, 0xc00232fdcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00232fc30 sp=0xc00232fc10 pc=0x4777ae +runtime.selectgo(0xc00232fe48, 0xc00232fdc8, 0xc00232fdd8?, 0x0, 0xc001d6e9e0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00232fd80 sp=0xc00232fc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001a0c410, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00232fe78 sp=0xc00232fd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001704d90) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00232ff30 sp=0xc00232fe78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc00232ffe0 sp=0xc00232ff30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00232ffe8 sp=0xc00232ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1210 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1168 [select]: +runtime.gopark(0xc00226fe48?, 0x2?, 0x0?, 0x0?, 0xc00226fdcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00226fc30 sp=0xc00226fc10 pc=0x4777ae +runtime.selectgo(0xc00226fe48, 0xc00226fdc8, 0xc0018b87e0?, 0x0, 0xc0006415a0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00226fd80 sp=0xc00226fc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00139d950, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00226fe78 sp=0xc00226fd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0018b87e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00226ff30 sp=0xc00226fe78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc00226ffe0 sp=0xc00226ff30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00226ffe8 sp=0xc00226ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1167 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1169 [select]: +runtime.gopark(0xc001821f70?, 0x4?, 0xd8?, 0xd7?, 0xc001821dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001821c00 sp=0xc001821be0 pc=0x4777ae +runtime.selectgo(0xc001821f70, 0xc001821dd0, 0x40a16a0?, 0x0, 0xc001bf03f0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001821d50 sp=0xc001821c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000b9e000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc001821fc0 sp=0xc001821d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc001821fe0 sp=0xc001821fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001821fe8 sp=0xc001821fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1167 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1218 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017337d8 sp=0xc0017337b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001733810 sp=0xc0017337d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494050, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001733830 sp=0xc001733810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00182c2a0, 0xc002168000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001733878 sp=0xc001733830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00182c280, {0xc002168000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001733968 sp=0xc001733878 pc=0x50e5a5 +net.(*netFD).Read(0xc00182c280, {0xc002168000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0017339c8 sp=0xc001733968 pc=0x89342b +net.(*conn).Read(0xc0006415a0, {0xc002168000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001733a50 sp=0xc0017339c8 pc=0x8b176d +net.(*TCPConn).Read(0xc0006415a0, {0xc002168000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc001733a98 sp=0xc001733a50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc0010bc300, {0xc000b9a2e0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001733b58 sp=0xc001733a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0010bc300}, {0xc000b9a2e0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001733ba0 sp=0xc001733b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000b9a2e0, 0x9, 0x9}, {0x5b0f0a0, 0xc0010bc300}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001733c40 sp=0xc001733ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000b9a2a0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001733d78 sp=0xc001733c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000b9e000, 0x28?, 0x5b426a8?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001733ec8 sp=0xc001733d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c1e0, {0x5b54e00?, 0xc000b9e000}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001733f80 sp=0xc001733ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001733fe0 sp=0xc001733f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001733fe8 sp=0xc001733fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1167 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1215 [select]: +runtime.gopark(0xc00181df70?, 0x4?, 0x0?, 0x0?, 0xc00181ddd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00181dc00 sp=0xc00181dbe0 pc=0x4777ae +runtime.selectgo(0xc00181df70, 0xc00181ddd0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00181dd50 sp=0xc00181dc00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001756680) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc00181dfc0 sp=0xc00181dd50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc00181dfe0 sp=0xc00181dfc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00181dfe8 sp=0xc00181dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1210 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1153 [select]: +runtime.gopark(0xc001f3df70?, 0x2?, 0xb4?, 0x1b?, 0xc001f3debc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f3dd20 sp=0xc001f3dd00 pc=0x4777ae +runtime.selectgo(0xc001f3df70, 0xc001f3deb8, 0x4aead9?, 0x0, 0x4b0571?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001f3de70 sp=0xc001f3dd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002785880, {0x5b426e0, 0xc0007c0690}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001f3dfa0 sp=0xc001f3de70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001f3dfe0 sp=0xc001f3dfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f3dfe8 sp=0xc001f3dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 609 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1234 [select]: +runtime.gopark(0xc001f3bf70?, 0x2?, 0xb4?, 0x1b?, 0xc001f3bebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f3bd20 sp=0xc001f3bd00 pc=0x4777ae +runtime.selectgo(0xc001f3bf70, 0xc001f3beb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001f3be70 sp=0xc001f3bd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0027858e0, {0x5b426e0, 0xc0007c06e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001f3bfa0 sp=0xc001f3be70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001f3bfe0 sp=0xc001f3bfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f3bfe8 sp=0xc001f3bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 609 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1356 [select]: +runtime.gopark(0xc001f39da8?, 0x2?, 0x60?, 0x98?, 0xc001f39a94?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f39818 sp=0xc001f397f8 pc=0x4777ae +runtime.selectgo(0xc001f39da8, 0xc001f39a90, 0x0?, 0x0, 0x1860a0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001f39968 sp=0xc001f39818 pc=0x48916b +github.com/hashicorp/go-plugin.Serve(0xc001e54ea0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:474 +0x2091 fp=0xc001f39fc0 sp=0xc001f39968 pc=0x21665b1 +github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins.func4() + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x34 fp=0xc001f39fe0 sp=0xc001f39fc0 pc=0x24cd234 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f39fe8 sp=0xc001f39fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins in goroutine 1320 + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x76b + +goroutine 1213 [select]: +runtime.gopark(0xc0022e7e70?, 0x2?, 0x0?, 0x0?, 0xc0022e7df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0022e7c58 sp=0xc0022e7c38 pc=0x4777ae +runtime.selectgo(0xc0022e7e70, 0xc0022e7df0, 0xc0022e7e00?, 0x0, 0xc001d6e9f8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0022e7da8 sp=0xc0022e7c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001a0c6e0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0022e7ea0 sp=0xc0022e7da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001704d20) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0022e7f58 sp=0xc0022e7ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc0022e7fe0 sp=0xc0022e7f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0022e7fe8 sp=0xc0022e7fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1151 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1268 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002747830 sp=0xc002747810 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002747868 sp=0xc002747830 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169d960, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002747888 sp=0xc002747868 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0027d59a0, 0xc002472000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0027478d0 sp=0xc002747888 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0027d5980, {0xc002472000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0027479c0 sp=0xc0027478d0 pc=0x50e5a5 +net.(*netFD).Read(0xc0027d5980, {0xc002472000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc002747a20 sp=0xc0027479c0 pc=0x89342b +net.(*conn).Read(0xc000bdffd0, {0xc002472000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc002747aa8 sp=0xc002747a20 pc=0x8b176d +net.(*TCPConn).Read(0xc000bdffd0, {0xc002472000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc002747af0 sp=0xc002747aa8 pc=0x8d72f4 +bufio.(*Reader).Read(0xc000d65440, {0xc001ebde00, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc002747bb0 sp=0xc002747af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000d65440}, {0xc001ebde00, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002747bf8 sp=0xc002747bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc001ebde00, 0x9, 0x9}, {0x5b0f0a0, 0xc000d65440}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc002747c98 sp=0xc002747bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001ebddc0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc002747dd0 sp=0xc002747c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00165a900, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc002747fb0 sp=0xc002747dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc002747fe0 sp=0xc002747fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002747fe8 sp=0xc002747fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1247 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 1269 [select]: +runtime.gopark(0xc002491e70?, 0x2?, 0x0?, 0x0?, 0xc002491df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002491c58 sp=0xc002491c38 pc=0x4777ae +runtime.selectgo(0xc002491e70, 0xc002491df0, 0xc002491e00?, 0x0, 0xc000bdffd0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002491da8 sp=0xc002491c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0007c1d10, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002491ea0 sp=0xc002491da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0023bcaf0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002491f58 sp=0xc002491ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc002491fe0 sp=0xc002491f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002491fe8 sp=0xc002491fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1247 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1216 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00172f7d8 sp=0xc00172f7b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00172f810 sp=0xc00172f7d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169df30, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00172f830 sp=0xc00172f810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0016f8da0, 0xc0022a8000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00172f878 sp=0xc00172f830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0016f8d80, {0xc0022a8000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00172f968 sp=0xc00172f878 pc=0x50e5a5 +net.(*netFD).Read(0xc0016f8d80, {0xc0022a8000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00172f9c8 sp=0xc00172f968 pc=0x89342b +net.(*conn).Read(0xc001d6e9e0, {0xc0022a8000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00172fa50 sp=0xc00172f9c8 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6e9e0, {0xc0022a8000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc00172fa98 sp=0xc00172fa50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc00150d980, {0xc0017030e0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00172fb58 sp=0xc00172fa98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00150d980}, {0xc0017030e0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00172fba0 sp=0xc00172fb58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0017030e0, 0x9, 0x9}, {0x5b0f0a0, 0xc00150d980}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00172fc40 sp=0xc00172fba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0017030a0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00172fd78 sp=0xc00172fc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001756680, 0x28?, 0x4258b60?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc00172fec8 sp=0xc00172fd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c1e0, {0x5b54e00?, 0xc001756680}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc00172ff80 sp=0xc00172fec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc00172ffe0 sp=0xc00172ff80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00172ffe8 sp=0xc00172ffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1210 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1242 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0022df830 sp=0xc0022df810 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0022df868 sp=0xc0022df830 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169de38, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0022df888 sp=0xc0022df868 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0027d57a0, 0xc002362000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0022df8d0 sp=0xc0022df888 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0027d5780, {0xc002362000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0022df9c0 sp=0xc0022df8d0 pc=0x50e5a5 +net.(*netFD).Read(0xc0027d5780, {0xc002362000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0022dfa20 sp=0xc0022df9c0 pc=0x89342b +net.(*conn).Read(0xc000bdff40, {0xc002362000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0022dfaa8 sp=0xc0022dfa20 pc=0x8b176d +net.(*TCPConn).Read(0xc000bdff40, {0xc002362000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc0022dfaf0 sp=0xc0022dfaa8 pc=0x8d72f4 +bufio.(*Reader).Read(0xc000d64de0, {0xc001ebdc40, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0022dfbb0 sp=0xc0022dfaf0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000d64de0}, {0xc001ebdc40, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0022dfbf8 sp=0xc0022dfbb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc001ebdc40, 0x9, 0x9}, {0x5b0f0a0, 0xc000d64de0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0022dfc98 sp=0xc0022dfbf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001ebdc00) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0022dfdd0 sp=0xc0022dfc98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00165a480, 0xc000d64e40?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0022dffb0 sp=0xc0022dfdd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0022dffe0 sp=0xc0022dffb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0022dffe8 sp=0xc0022dffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1235 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 1243 [select]: +runtime.gopark(0xc0023b5e70?, 0x2?, 0x0?, 0x0?, 0xc0023b5df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0023b5c58 sp=0xc0023b5c38 pc=0x4777ae +runtime.selectgo(0xc0023b5e70, 0xc0023b5df0, 0xc0023b5e00?, 0x0, 0xc000bdff40?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0023b5da8 sp=0xc0023b5c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0007c1590, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0023b5ea0 sp=0xc0023b5da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001cf2c40) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0023b5f58 sp=0xc0023b5ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc0023b5fe0 sp=0xc0023b5f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0023b5fe8 sp=0xc0023b5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1235 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1244 [select]: +runtime.gopark(0xc0023ade48?, 0x2?, 0x0?, 0x0?, 0xc0023addcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0023adc30 sp=0xc0023adc10 pc=0x4777ae +runtime.selectgo(0xc0023ade48, 0xc0023addc8, 0xc0023addd8?, 0x0, 0xc000bdff28?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0023add80 sp=0xc0023adc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0007c1680, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0023ade78 sp=0xc0023add80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001cf2cb0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0023adf30 sp=0xc0023ade78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc0023adfe0 sp=0xc0023adf30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0023adfe8 sp=0xc0023adfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1240 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1245 [select]: +runtime.gopark(0xc00210bf70?, 0x4?, 0xa9?, 0x5?, 0xc00210bdd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00210bc00 sp=0xc00210bbe0 pc=0x4777ae +runtime.selectgo(0xc00210bf70, 0xc00210bdd0, 0x5a00000000000000?, 0x0, 0x4?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00210bd50 sp=0xc00210bc00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001f4aea0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc00210bfc0 sp=0xc00210bd50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc00210bfe0 sp=0xc00210bfc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00210bfe8 sp=0xc00210bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1240 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1246 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f077d8 sp=0xc001f077b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001f07810 sp=0xc001f077d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169dd40, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001f07830 sp=0xc001f07810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0027d5820, 0xc00236e000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001f07878 sp=0xc001f07830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0027d5800, {0xc00236e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001f07968 sp=0xc001f07878 pc=0x50e5a5 +net.(*netFD).Read(0xc0027d5800, {0xc00236e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001f079c8 sp=0xc001f07968 pc=0x89342b +net.(*conn).Read(0xc000bdff28, {0xc00236e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001f07a50 sp=0xc001f079c8 pc=0x8b176d +net.(*TCPConn).Read(0xc000bdff28, {0xc00236e000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc001f07a98 sp=0xc001f07a50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc000d64ea0, {0xc001ebdd20, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001f07b58 sp=0xc001f07a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000d64ea0}, {0xc001ebdd20, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001f07ba0 sp=0xc001f07b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc001ebdd20, 0x9, 0x9}, {0x5b0f0a0, 0xc000d64ea0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001f07c40 sp=0xc001f07ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001ebdce0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001f07d78 sp=0xc001f07c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001f4aea0, 0x4208e0?, 0x7f429a46a200?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001f07ec8 sp=0xc001f07d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c1e0, {0x5b54e00?, 0xc001f4aea0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001f07f80 sp=0xc001f07ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001f07fe0 sp=0xc001f07f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f07fe8 sp=0xc001f07fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1240 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1383 [IO wait]: +runtime.gopark(0xc0024d4fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024d4f98 sp=0xc0024d4f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0024d4fd0 sp=0xc0024d4f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c49a0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0024d4ff0 sp=0xc0024d4fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0024adea0, 0xc0022f9200?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0024d5038 sp=0xc0024d4ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0024ade80, {0xc0022f9200, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0024d5128 sp=0xc0024d5038 pc=0x50e5a5 +net.(*netFD).Read(0xc0024ade80, {0xc0022f9200, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0024d5188 sp=0xc0024d5128 pc=0x89342b +net.(*conn).Read(0xc0012c7398, {0xc0022f9200, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0024d5210 sp=0xc0024d5188 pc=0x8b176d +net.(*UnixConn).Read(0xc0012c7398, {0xc0022f9200, 0x600, 0x600}) + :1 +0x54 fp=0xc0024d5258 sp=0xc0024d5210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc00310cf18, {0xc0022f9200, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0024d52b8 sp=0xc0024d5258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000f9db28, {0x5b0f6c0, 0xc00310cf18}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0024d5320 sp=0xc0024d52b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000f9d880, {0x5b0f840?, 0xc0012c7398}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc0024d5390 sp=0xc0024d5320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000f9d880, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0024d59c0 sp=0xc0024d5390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000f9d880, {0xc002810000, 0x8000, 0xc0024d5af8?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc0024d5aa8 sp=0xc0024d59c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001d86de0, {0xc002810000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc0024d5af0 sp=0xc0024d5aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc002175680, {0xc0014f9000, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0024d5bb0 sp=0xc0024d5af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc002175680}, {0xc0014f9000, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0024d5bf8 sp=0xc0024d5bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0014f9000, 0x9, 0x9}, {0x5b0f0a0, 0xc002175680}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0024d5c98 sp=0xc0024d5bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0014f8fc0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0024d5dd0 sp=0xc0024d5c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001455d40, 0xc000f9d880?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0024d5fb0 sp=0xc0024d5dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0024d5fe0 sp=0xc0024d5fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024d5fe8 sp=0xc0024d5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1380 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 1251 [IO wait]: +runtime.gopark(0x4?, 0xc0017a7c30?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017a7c48 sp=0xc0017a7c28 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0017a7c80 sp=0xc0017a7c48 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169dc48, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0017a7ca0 sp=0xc0017a7c80 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001655280, 0xc002424000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0017a7ce8 sp=0xc0017a7ca0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001655260, {0xc002424000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0017a7dd8 sp=0xc0017a7ce8 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc001d6eac0, {0xc002424000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc0017a7e38 sp=0xc0017a7dd8 pc=0x51ed0d +io.copyBuffer({0x5b0ede0, 0xc0010d91d0}, {0x5b0eb40, 0xc001d6eac0}, {0x0, 0x0, 0x0}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:430 +0x27e fp=0xc0017a7ee0 sp=0xc0017a7e38 pc=0x50115e +io.Copy(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:389 +os/exec.(*Cmd).writerDescriptor.func1() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:560 +0x5b fp=0xc0017a7f50 sp=0xc0017a7ee0 pc=0x6552db +os/exec.(*Cmd).Start.func2(0xc001635e00) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:717 +0x3d fp=0xc0017a7fb8 sp=0xc0017a7f50 pc=0x65669d +os/exec.(*Cmd).Start.func3() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:729 +0x42 fp=0xc0017a7fe0 sp=0xc0017a7fb8 pc=0x656622 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017a7fe8 sp=0xc0017a7fe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 1250 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:716 +0xf3a + +goroutine 1252 [IO wait]: +runtime.gopark(0x4b05a9?, 0xc00274bc70?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00274bc48 sp=0xc00274bc28 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00274bc80 sp=0xc00274bc48 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169da58, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00274bca0 sp=0xc00274bc80 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001655340, 0xc00244a000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00274bce8 sp=0xc00274bca0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001655320, {0xc00244a000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00274bdd8 sp=0xc00274bce8 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc001d6eaf8, {0xc00244a000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc00274be38 sp=0xc00274bdd8 pc=0x51ed0d +io.copyBuffer({0x5b0ede0, 0xc0010d91e8}, {0x5b0eb40, 0xc001d6eaf8}, {0x0, 0x0, 0x0}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:430 +0x27e fp=0xc00274bee0 sp=0xc00274be38 pc=0x50115e +io.Copy(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:389 +os/exec.(*Cmd).writerDescriptor.func1() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:560 +0x5b fp=0xc00274bf50 sp=0xc00274bee0 pc=0x6552db +os/exec.(*Cmd).Start.func2(0xc001635e60) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:717 +0x3d fp=0xc00274bfb8 sp=0xc00274bf50 pc=0x65669d +os/exec.(*Cmd).Start.func3() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:729 +0x42 fp=0xc00274bfe0 sp=0xc00274bfb8 pc=0x656622 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00274bfe8 sp=0xc00274bfe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 1250 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:716 +0xf3a + +goroutine 1253 [select]: +runtime.gopark(0xc001de54b0?, 0x656a30?, 0x0?, 0x1?, 0xc001b6ce7c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b6cce0 sp=0xc001b6ccc0 pc=0x4777ae +runtime.selectgo(0xc001b6cf60, 0xc001b6ce78, 0x0?, 0x1, 0x40a16a0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b6ce30 sp=0xc001b6cce0 pc=0x48916b +os/exec.(*Cmd).watchCtx(0xc00116e160, 0xc001de54a0) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:757 +0x10f fp=0xc001b6cfb0 sp=0xc001b6ce30 pc=0x656a6f +os/exec.(*Cmd).Start.func4() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:743 +0x45 fp=0xc001b6cfe0 sp=0xc001b6cfb0 pc=0x6565a5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b6cfe8 sp=0xc001b6cfe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 1250 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:743 +0xf05 + +goroutine 1254 [syscall]: +syscall.Syscall6(0x0?, 0x0?, 0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/syscall/syscall_linux.go:91 +0x3d fp=0xc002103d38 sp=0xc002103cb0 pc=0x4d1b1d +os.(*Process).blockUntilWaitable(0xc001620c90) + /home/ec2-user/sdk/go1.21.0/src/os/wait_waitid.go:32 +0x9f fp=0xc002103e18 sp=0xc002103d38 pc=0x529c5f +os.(*Process).wait(0xc001620c90) + /home/ec2-user/sdk/go1.21.0/src/os/exec_unix.go:22 +0x4b fp=0xc002103e98 sp=0xc002103e18 pc=0x51e0ab +os.(*Process).Wait(...) + /home/ec2-user/sdk/go1.21.0/src/os/exec.go:134 +os/exec.(*Cmd).Wait(0xc00116e160) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:890 +0xb9 fp=0xc002103f60 sp=0xc002103e98 pc=0x657279 +github.com/onsi/gomega/gexec.(*Session).monitorForExit(0xc002377380, 0x32594b527a594f52?) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:196 +0x3f fp=0xc002103fb0 sp=0xc002103f60 pc=0x233731f +github.com/onsi/gomega/gexec.Start.func1() + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:96 +0x45 fp=0xc002103fe0 sp=0xc002103fb0 pc=0x2336d65 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002103fe8 sp=0xc002103fe0 pc=0x4ad421 +created by github.com/onsi/gomega/gexec.Start in goroutine 1250 + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:96 +0x9eb + +goroutine 1258 [select]: +runtime.gopark(0xc0025a7e48?, 0x2?, 0x0?, 0x0?, 0xc0025a7dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0025a7c30 sp=0xc0025a7c10 pc=0x4777ae +runtime.selectgo(0xc0025a7e48, 0xc0025a7dc8, 0xc0025a7dd8?, 0x0, 0xc001d6eb40?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0025a7d80 sp=0xc0025a7c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001a0d400, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0025a7e78 sp=0xc0025a7d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001705110) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0025a7f30 sp=0xc0025a7e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc0025a7fe0 sp=0xc0025a7f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0025a7fe8 sp=0xc0025a7fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1257 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1259 [select]: +runtime.gopark(0xc00181ef70?, 0x4?, 0x20?, 0x97?, 0xc00181edd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00181ec00 sp=0xc00181ebe0 pc=0x4777ae +runtime.selectgo(0xc00181ef70, 0xc00181edd0, 0x420000c0017030c0?, 0x0, 0x5?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00181ed50 sp=0xc00181ec00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0017576c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc00181efc0 sp=0xc00181ed50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc00181efe0 sp=0xc00181efc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00181efe8 sp=0xc00181efe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1257 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1260 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00181c7d8 sp=0xc00181c7b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00181c810 sp=0xc00181c7d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169db50, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00181c830 sp=0xc00181c810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0016f92a0, 0xc002580000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00181c878 sp=0xc00181c830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0016f9280, {0xc002580000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00181c968 sp=0xc00181c878 pc=0x50e5a5 +net.(*netFD).Read(0xc0016f9280, {0xc002580000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00181c9c8 sp=0xc00181c968 pc=0x89342b +net.(*conn).Read(0xc001d6eb40, {0xc002580000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00181ca50 sp=0xc00181c9c8 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6eb40, {0xc002580000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc00181ca98 sp=0xc00181ca50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc001655ce0, {0xc0017032a0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00181cb58 sp=0xc00181ca98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001655ce0}, {0xc0017032a0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00181cba0 sp=0xc00181cb58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0017032a0, 0x9, 0x9}, {0x5b0f0a0, 0xc001655ce0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00181cc40 sp=0xc00181cba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001703260) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00181cd78 sp=0xc00181cc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0017576c0, 0x5b423c8?, 0x74736f686c000000?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc00181cec8 sp=0xc00181cd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c1e0, {0x5b54e00?, 0xc0017576c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc00181cf80 sp=0xc00181cec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc00181cfe0 sp=0xc00181cf80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00181cfe8 sp=0xc00181cfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1257 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1310 [select]: +runtime.gopark(0xc002635e70?, 0x2?, 0x0?, 0x0?, 0xc002635df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002635c58 sp=0xc002635c38 pc=0x4777ae +runtime.selectgo(0xc002635e70, 0xc002635df0, 0xc002635e00?, 0x0, 0xc00133a000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002635da8 sp=0xc002635c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001c65db0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002635ea0 sp=0xc002635da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0026547e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002635f58 sp=0xc002635ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc002635fe0 sp=0xc002635f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002635fe8 sp=0xc002635fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1353 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1314 [IO wait]: +runtime.gopark(0x4?, 0xc000d02c30?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d02c48 sp=0xc000d02c28 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000d02c80 sp=0xc000d02c48 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169d770, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000d02ca0 sp=0xc000d02c80 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00128a7a0, 0xc00249a000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000d02ce8 sp=0xc000d02ca0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00128a780, {0xc00249a000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc000d02dd8 sp=0xc000d02ce8 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc0012c6dc0, {0xc00249a000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc000d02e38 sp=0xc000d02dd8 pc=0x51ed0d +io.copyBuffer({0x5b0ede0, 0xc0013200c0}, {0x5b0eb40, 0xc0012c6dc0}, {0x0, 0x0, 0x0}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:430 +0x27e fp=0xc000d02ee0 sp=0xc000d02e38 pc=0x50115e +io.Copy(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:389 +os/exec.(*Cmd).writerDescriptor.func1() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:560 +0x5b fp=0xc000d02f50 sp=0xc000d02ee0 pc=0x6552db +os/exec.(*Cmd).Start.func2(0xc00127dcc0) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:717 +0x3d fp=0xc000d02fb8 sp=0xc000d02f50 pc=0x65669d +os/exec.(*Cmd).Start.func3() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:729 +0x42 fp=0xc000d02fe0 sp=0xc000d02fb8 pc=0x656622 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d02fe8 sp=0xc000d02fe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 1250 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:716 +0xf3a + +goroutine 1355 [chan receive]: +runtime.gopark(0xc001f48830?, 0xffffffff?, 0xa9?, 0x5?, 0xc001ef3f40?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ef3f08 sp=0xc001ef3ee8 pc=0x4777ae +runtime.chanrecv(0xc001089680, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001ef3f80 sp=0xc001ef3f08 pc=0x441325 +runtime.chanrecv1(0xc001b6e730?, 0xc001ef3fd0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001ef3fa8 sp=0xc001ef3f80 pc=0x440f72 +github.com/rancher/opni/pkg/clients.NewGatewayClient.func1() + /home/ec2-user/opni/pkg/clients/gateway_client.go:69 +0x4a fp=0xc001ef3fe0 sp=0xc001ef3fa8 pc=0x200c96a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ef3fe8 sp=0xc001ef3fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/clients.NewGatewayClient in goroutine 1320 + /home/ec2-user/opni/pkg/clients/gateway_client.go:68 +0x2dd + +goroutine 1379 [select]: +runtime.gopark(0xc0024e1f70?, 0x2?, 0xb4?, 0x1b?, 0xc0024e1ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024e1d20 sp=0xc0024e1d00 pc=0x4777ae +runtime.selectgo(0xc0024e1f70, 0xc0024e1eb8, 0x0?, 0x0, 0xc0001261c8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0024e1e70 sp=0xc0024e1d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001d86ae0, {0x5b426e0, 0xc002338cd0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0024e1fa0 sp=0xc0024e1e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0024e1fe0 sp=0xc0024e1fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024e1fe8 sp=0xc0024e1fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1361 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1316 [select]: +runtime.gopark(0xc00123e790?, 0x656a30?, 0x0?, 0x1?, 0xc001983e7c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001983ce0 sp=0xc001983cc0 pc=0x4777ae +runtime.selectgo(0xc001983f60, 0xc001983e78, 0x4b05a9?, 0x1, 0x1f76a9f?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001983e30 sp=0xc001983ce0 pc=0x48916b +os/exec.(*Cmd).watchCtx(0xc0026c3340, 0xc00123e780) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:757 +0x10f fp=0xc001983fb0 sp=0xc001983e30 pc=0x656a6f +os/exec.(*Cmd).Start.func4() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:743 +0x45 fp=0xc001983fe0 sp=0xc001983fb0 pc=0x6565a5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001983fe8 sp=0xc001983fe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 1250 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:743 +0xf05 + +goroutine 1315 [IO wait]: +runtime.gopark(0x4b05a9?, 0xc002100c70?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002100c48 sp=0xc002100c28 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002100c80 sp=0xc002100c48 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169d488, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002100ca0 sp=0xc002100c80 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00128a860, 0xc002610000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002100ce8 sp=0xc002100ca0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00128a840, {0xc002610000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc002100dd8 sp=0xc002100ce8 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc0012c6e00, {0xc002610000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc002100e38 sp=0xc002100dd8 pc=0x51ed0d +io.copyBuffer({0x5b0ede0, 0xc0013200d8}, {0x5b0eb40, 0xc0012c6e00}, {0x0, 0x0, 0x0}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:430 +0x27e fp=0xc002100ee0 sp=0xc002100e38 pc=0x50115e +io.Copy(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:389 +os/exec.(*Cmd).writerDescriptor.func1() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:560 +0x5b fp=0xc002100f50 sp=0xc002100ee0 pc=0x6552db +os/exec.(*Cmd).Start.func2(0xc00127dd60) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:717 +0x3d fp=0xc002100fb8 sp=0xc002100f50 pc=0x65669d +os/exec.(*Cmd).Start.func3() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:729 +0x42 fp=0xc002100fe0 sp=0xc002100fb8 pc=0x656622 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002100fe8 sp=0xc002100fe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 1250 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:716 +0xf3a + +goroutine 1446 [select]: +runtime.gopark(0xc001086fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001086f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001086de8 sp=0xc001086dc8 pc=0x4777ae +runtime.selectgo(0xc001086fb0, 0xc001086f70, 0x0?, 0x0, 0x6000103?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001086f38 sp=0xc001086de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001086fe0 sp=0xc001086f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001086fe8 sp=0xc001086fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1361 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1389 [select]: +runtime.gopark(0xc00260ffb0?, 0x2?, 0xb0?, 0xfd?, 0xc00260fee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00260fd58 sp=0xc00260fd38 pc=0x4777ae +runtime.selectgo(0xc00260ffb0, 0xc00260fee0, 0xc00260ff98?, 0x0, 0xc0027107b0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00260fea8 sp=0xc00260fd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc00260ffe0 sp=0xc00260fea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00260ffe8 sp=0xc00260ffe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1444 [select]: +runtime.gopark(0xc001e22890?, 0x2?, 0x98?, 0x27?, 0xc001e2283c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e226a8 sp=0xc001e22688 pc=0x4777ae +runtime.selectgo(0xc001e22890, 0xc001e22838, 0x7f429a489100?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e227f8 sp=0xc001e226a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002658460, {0xc002652c70, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc001e228c0 sp=0xc001e227f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002658460, {0xc002652c70, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001e22958 sp=0xc001e228c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002652bd0, {0xc002652c70, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001e229a0 sp=0xc001e22958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002652bd0}, {0xc002652c70, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001e229e8 sp=0xc001e229a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0018f4fc0, {0xc002652c70, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001e22a58 sp=0xc001e229e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002652c60, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001e22b28 sp=0xc001e22a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc002171500?, 0xc0018f4fc0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001e22c18 sp=0xc001e22b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002652c60, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc002658780}, 0xc0025ccf60?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001e22cd0 sp=0xc001e22c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000b9c0f0, {0x43dcf60?, 0xc002658780}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001e22f08 sp=0xc001e22cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002652db0, {0x43dcf60, 0xc002658780}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001e22ff0 sp=0xc001e22f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0015ffd20, {0x43dcf60, 0xc002658780}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001e23028 sp=0xc001e22ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00271e250) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001e23068 sp=0xc001e23028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00271e250?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc001e230a8 sp=0xc001e23068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00271e260) + :1 +0x46 fp=0xc001e230e0 sp=0xc001e230a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0025cd0e0, {0x5b51368, 0xc00271e260}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001e23320 sp=0xc001e230e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00234fbc0, {0x5b51038, 0xc00271e250}) + :1 +0xaf fp=0xc001e23370 sp=0xc001e23320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00234fbc0}, {0x5b4a6f0?, 0xc0015ffd20}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc001e233b0 sp=0xc001e23370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00234fbc0}, {0x5b4ab30, 0xc002652db0}, 0xc000580d38?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001e23400 sp=0xc001e233b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00234fbc0}, {0x5b4ab30, 0xc002652db0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001e23490 sp=0xc001e23400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00234fbc0}, {0x5b492f8, 0xc000b9c0f0}, 0xc000580d38, 0xc002170c40) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001e237d0 sp=0xc001e23490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00234fbc0}, {0x5b492f8, 0xc000b9c0f0}, 0x3f252a0?, 0xc00271e180?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001e23848 sp=0xc001e237d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc0018f4fc0, 0xc0025cd170, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001e23db0 sp=0xc001e23848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc0018f4fc0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001e23f28 sp=0xc001e23db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001e23fe0 sp=0xc001e23f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e23fe8 sp=0xc001e23fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1387 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1312 [select]: +runtime.gopark(0xc0015e7fb0?, 0x2?, 0xb4?, 0x1b?, 0xc0015e7f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015e7de8 sp=0xc0015e7dc8 pc=0x4777ae +runtime.selectgo(0xc0015e7fb0, 0xc0015e7f70, 0x0?, 0x0, 0xc0026a43c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0015e7f38 sp=0xc0015e7de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0015e7fe0 sp=0xc0015e7f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015e7fe8 sp=0xc0015e7fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1361 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1390 [chan receive]: +runtime.gopark(0x40a16a0?, 0x0?, 0x40?, 0x6e?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002101d60 sp=0xc002101d40 pc=0x4777ae +runtime.chanrecv(0xc0022fb260, 0xc002101e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002101dd8 sp=0xc002101d60 pc=0x441325 +runtime.chanrecv1(0xc002339618?, 0x40a16a0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002101e00 sp=0xc002101dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002101fe0 sp=0xc002101e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002101fe8 sp=0xc002101fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1384 [select]: +runtime.gopark(0xc002987e70?, 0x2?, 0x0?, 0x0?, 0xc002987df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002987c58 sp=0xc002987c38 pc=0x4777ae +runtime.selectgo(0xc002987e70, 0xc002987df0, 0xc002987e00?, 0x0, 0xc000f9d880?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002987da8 sp=0xc002987c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0023394f0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002987ea0 sp=0xc002987da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002714a80) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002987f58 sp=0xc002987ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc002987fe0 sp=0xc002987f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002987fe8 sp=0xc002987fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1380 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1385 [select]: +runtime.gopark(0xc002879e48?, 0x2?, 0x0?, 0x0?, 0xc002879dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002879c30 sp=0xc002879c10 pc=0x4777ae +runtime.selectgo(0xc002879e48, 0xc002879dc8, 0xc002879dd8?, 0x0, 0xc00133b180?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002879d80 sp=0xc002879c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0023395e0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002879e78 sp=0xc002879d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002714bd0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002879f30 sp=0xc002879e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc002879fe0 sp=0xc002879f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002879fe8 sp=0xc002879fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1394 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1386 [select]: +runtime.gopark(0xc001987f70?, 0x4?, 0x10?, 0x6d?, 0xc001987dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001987c00 sp=0xc001987be0 pc=0x4777ae +runtime.selectgo(0xc001987f70, 0xc001987dd0, 0xc001987d80?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001987d50 sp=0xc001987c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0024e7860) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc001987fc0 sp=0xc001987d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc001987fe0 sp=0xc001987fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001987fe8 sp=0xc001987fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1394 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1313 [select]: +runtime.gopark(0xc001e3afb0?, 0x2?, 0xb0?, 0xad?, 0xc001e3aee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e3ad58 sp=0xc001e3ad38 pc=0x4777ae +runtime.selectgo(0xc001e3afb0, 0xc001e3aee0, 0x1128d01?, 0x0, 0xffffffffffffffff?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e3aea8 sp=0xc001e3ad58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001e3afe0 sp=0xc001e3aea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e3afe8 sp=0xc001e3afe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1357 [select]: +runtime.gopark(0xc001d49f48?, 0x2?, 0x50?, 0x9d?, 0xc001d49f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d49d80 sp=0xc001d49d60 pc=0x4777ae +runtime.selectgo(0xc001d49f48, 0xc001d49f00, 0xc001d49f08?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001d49ed0 sp=0xc001d49d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).Recv(0xc0025cd230) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:124 +0xeb fp=0xc001d49f78 sp=0xc001d49ed0 pc=0x2154feb +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc001abbc70) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc001d49fc0 sp=0xc001d49f78 pc=0x215759f +github.com/hashicorp/go-plugin.(*GRPCServer).Init.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x34 fp=0xc001d49fe0 sp=0xc001d49fc0 pc=0x215adf4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d49fe8 sp=0xc001d49fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*GRPCServer).Init in goroutine 1356 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x89d + +goroutine 1358 [IO wait]: +runtime.gopark(0x2?, 0x48eaa0?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00215eb30 sp=0xc00215eb10 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00215eb68 sp=0xc00215eb30 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169d1a0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00215eb88 sp=0xc00215eb68 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001e54fe0, 0xc00271a000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00215ebd0 sp=0xc00215eb88 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001e54fc0, {0xc00271a000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00215ecc0 sp=0xc00215ebd0 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc001f4e390, {0xc00271a000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc00215ed20 sp=0xc00215ecc0 pc=0x51ed0d +io.(*teeReader).Read(0xc002180940, {0xc00271a000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:620 +0x5d fp=0xc00215ed80 sp=0xc00215ed20 pc=0x501c5d +bufio.(*Reader).Read(0xc00215ef18, {0xc0024cf800, 0x400, 0x400}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00215ee40 sp=0xc00215ed80 pc=0x5f943e +github.com/hashicorp/go-plugin.copyChan({0x5b611c8, 0xc0025c2c60}, 0xc001d9d440?, {0x5b0ee40?, 0xc002180940}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:184 +0x1f4 fp=0xc00215ef80 sp=0xc00215ee40 pc=0x215ccf4 +github.com/hashicorp/go-plugin.newGRPCStdioServer.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:40 +0x6c fp=0xc00215efe0 sp=0xc00215ef80 pc=0x215b9ac +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00215efe8 sp=0xc00215efe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCStdioServer in goroutine 1356 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:40 +0x169 + +goroutine 1359 [IO wait]: +runtime.gopark(0x4b05a9?, 0xc00260bb60?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00260bb30 sp=0xc00260bb10 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00260bb68 sp=0xc00260bb30 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c5068, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00260bb88 sp=0xc00260bb68 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001e550a0, 0xc0024b9000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00260bbd0 sp=0xc00260bb88 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001e55080, {0xc0024b9000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00260bcc0 sp=0xc00260bbd0 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc001f4e3a0, {0xc0024b9000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc00260bd20 sp=0xc00260bcc0 pc=0x51ed0d +io.(*teeReader).Read(0xc002180960, {0xc0024b9000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:620 +0x5d fp=0xc00260bd80 sp=0xc00260bd20 pc=0x501c5d +bufio.(*Reader).Read(0xc00260bf18, {0xc0024cf400, 0x400, 0x400}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00260be40 sp=0xc00260bd80 pc=0x5f943e +github.com/hashicorp/go-plugin.copyChan({0x5b611c8, 0xc0025c2c60}, 0xc001d9d4a0?, {0x5b0ee40?, 0xc002180960}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:184 +0x1f4 fp=0xc00260bf80 sp=0xc00260be40 pc=0x215ccf4 +github.com/hashicorp/go-plugin.newGRPCStdioServer.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:41 +0x6c fp=0xc00260bfe0 sp=0xc00260bf80 pc=0x215b90c +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00260bfe8 sp=0xc00260bfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCStdioServer in goroutine 1356 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:41 +0x265 + +goroutine 1360 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024cd940 sp=0xc0024cd920 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0024cd978 sp=0xc0024cd940 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169d678, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0024cd998 sp=0xc0024cd978 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0025f2320, 0xc0024cda01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0024cd9e0 sp=0xc0024cd998 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0025f2300) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0024cdad0 sp=0xc0024cd9e0 pc=0x513f65 +net.(*netFD).accept(0xc0025f2300) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0024cdbc8 sp=0xc0024cdad0 pc=0x8967be +net.(*UnixListener).accept(0xc0025cce70) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc0024cdc08 sp=0xc0024cdbc8 pc=0x8cedfe +net.(*UnixListener).Accept(0xc0025cce70) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc0024cdc78 sp=0xc0024cdc08 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc002180920) + :1 +0x44 fp=0xc0024cdcb0 sp=0xc0024cdc78 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc0018534a0, {0x5b2c6a0, 0xc002180920}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc0024cdee0 sp=0xc0024cdcb0 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCServer).Serve(0xc0025c2d80, {0x5b2c6a0, 0xc002180920}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:153 +0xdd fp=0xc0024cdf98 sp=0xc0024cdee0 pc=0x215b33d +github.com/hashicorp/go-plugin.Serve.func6() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x62 fp=0xc0024cdfe0 sp=0xc0024cdf98 pc=0x2166b62 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024cdfe8 sp=0xc0024cdfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.Serve in goroutine 1356 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x1f67 + +goroutine 1453 [select]: +runtime.gopark(0xc002666310?, 0x13b0513?, 0x0?, 0x1?, 0xc0029b1ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029b1d30 sp=0xc0029b1d10 pc=0x4777ae +runtime.selectgo(0xc0029b1f60, 0xc0029b1ec8, 0xc0029b1eb8?, 0x1, 0x28?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029b1e80 sp=0xc0029b1d30 pc=0x48916b +github.com/lestrrat-go/backoff/v2.(*controller).loop(0xc00251d020) + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:64 +0x318 fp=0xc0029b1fc0 sp=0xc0029b1e80 pc=0x13b0538 +github.com/lestrrat-go/backoff/v2.newController.func1() + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x34 fp=0xc0029b1fe0 sp=0xc0029b1fc0 pc=0x13b01f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029b1fe8 sp=0xc0029b1fe0 pc=0x4ad421 +created by github.com/lestrrat-go/backoff/v2.newController in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x4dd + +goroutine 1362 [select]: +runtime.gopark(0xc0029e7da8?, 0x2?, 0x60?, 0x78?, 0xc0029e7a94?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029e7818 sp=0xc0029e77f8 pc=0x4777ae +runtime.selectgo(0xc0029e7da8, 0xc0029e7a90, 0x0?, 0x0, 0x1860a0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029e7968 sp=0xc0029e7818 pc=0x48916b +github.com/hashicorp/go-plugin.Serve(0xc001e55500) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:474 +0x2091 fp=0xc0029e7fc0 sp=0xc0029e7968 pc=0x21665b1 +github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins.func4() + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x34 fp=0xc0029e7fe0 sp=0xc0029e7fc0 pc=0x24cd234 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029e7fe8 sp=0xc0029e7fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins in goroutine 1320 + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x76b + +goroutine 1294 [select]: +runtime.gopark(0xc0025eb748?, 0x2?, 0xd0?, 0xb5?, 0xc0025eb6fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0025eb578 sp=0xc0025eb558 pc=0x4777ae +runtime.selectgo(0xc0025eb748, 0xc0025eb6f8, 0xc0021827c0?, 0x0, 0xc00257d798?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0025eb6c8 sp=0xc0025eb578 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c6b200) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc0025eb778 sp=0xc0025eb6c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc0026a88f0, {0x43dd860?, 0xc0028460a0}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0025eb9d8 sp=0xc0025eb778 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001d569b4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc0025eba38 sp=0xc0025eb9d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001d56900, 0xc0025ebbf0, 0xc0025ebbc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc0025ebae0 sp=0xc0025eba38 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d56900, {0x43dd860, 0xc0028460a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc0025ebc20 sp=0xc0025ebae0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc0021e6200, {0x43dd860, 0xc0028460a0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc0025ebde8 sp=0xc0025ebc20 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamClient).Recv(0xc0025771b0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:132 +0x66 fp=0xc0025ebe28 sp=0xc0025ebde8 pc=0x2102606 +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream(0xc001ff19c0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:197 +0x2e8 fp=0xc0025ebfc0 sp=0xc0025ebe28 pc=0x2155468 +github.com/hashicorp/go-plugin.newGRPCClient.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x34 fp=0xc0025ebfe0 sp=0xc0025ebfc0 pc=0x2159954 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0025ebfe8 sp=0xc0025ebfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 1361 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x62e + +goroutine 1295 [select]: +runtime.gopark(0xc002795e48?, 0x2?, 0x0?, 0x0?, 0xc002795dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002795c30 sp=0xc002795c10 pc=0x4777ae +runtime.selectgo(0xc002795e48, 0xc002795dc8, 0xc00257b7a0?, 0x0, 0x5?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002795d80 sp=0xc002795c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0026a4e10, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002795e78 sp=0xc002795d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00257b7a0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002795f30 sp=0xc002795e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc002795fe0 sp=0xc002795f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002795fe8 sp=0xc002795fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1344 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1296 [select]: +runtime.gopark(0xc001820f70?, 0x4?, 0xa9?, 0x5?, 0xc001820dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001820c00 sp=0xc001820be0 pc=0x4777ae +runtime.selectgo(0xc001820f70, 0xc001820dd0, 0x40a16a0?, 0x0, 0xc001fafff0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001820d50 sp=0xc001820c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002515040) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc001820fc0 sp=0xc001820d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc001820fe0 sp=0xc001820fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001820fe8 sp=0xc001820fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1344 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1297 [IO wait]: +runtime.gopark(0xc0024dd048?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024dd000 sp=0xc0024dcfe0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0024dd038 sp=0xc0024dd000 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169d298, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0024dd058 sp=0xc0024dd038 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0024adba0, 0xc002e2ea00?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0024dd0a0 sp=0xc0024dd058 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0024adb80, {0xc002e2ea00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0024dd190 sp=0xc0024dd0a0 pc=0x50e5a5 +net.(*netFD).Read(0xc0024adb80, {0xc002e2ea00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0024dd1f0 sp=0xc0024dd190 pc=0x89342b +net.(*conn).Read(0xc0012c72e0, {0xc002e2ea00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0024dd278 sp=0xc0024dd1f0 pc=0x8b176d +net.(*TCPConn).Read(0xc0012c72e0, {0xc002e2ea00, 0x500, 0x500}) + :1 +0x54 fp=0xc0024dd2c0 sp=0xc0024dd278 pc=0x8d72f4 +crypto/tls.(*atLeastReader).Read(0xc00304a2a0, {0xc002e2ea00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0024dd320 sp=0xc0024dd2c0 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000f9d428, {0x5b0f6c0, 0xc00304a2a0}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0024dd388 sp=0xc0024dd320 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000f9d180, {0x5b0f820?, 0xc0012c72e0}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc0024dd3f8 sp=0xc0024dd388 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000f9d180, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0024dda28 sp=0xc0024dd3f8 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000f9d180, {0xc002686580, 0x9, 0x4aef8c?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc0024ddb10 sp=0xc0024dda28 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0026e0240, {0xc002686580, 0x9, 0x9}) + :1 +0x6d fp=0xc0024ddb58 sp=0xc0024ddb10 pc=0x10969ed +io.ReadAtLeast({0x7f429a510838, 0xc0026e0240}, {0xc002686580, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0024ddba0 sp=0xc0024ddb58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc002686580, 0x9, 0x9}, {0x7f429a510838, 0xc0026e0240}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0024ddc40 sp=0xc0024ddba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc002686540) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0024ddd78 sp=0xc0024ddc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002515040, 0x2020200a3936202d?, 0x200a313131202d20?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc0024ddec8 sp=0xc0024ddd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00170e000, {0x5b54e00?, 0xc002515040}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc0024ddf80 sp=0xc0024ddec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc0024ddfe0 sp=0xc0024ddf80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024ddfe8 sp=0xc0024ddfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1344 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1363 [select]: +runtime.gopark(0xc001487f48?, 0x2?, 0xc8?, 0xf1?, 0xc001487f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001487d80 sp=0xc001487d60 pc=0x4777ae +runtime.selectgo(0xc001487f48, 0xc001487f00, 0xc001487f08?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001487ed0 sp=0xc001487d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).Recv(0xc0025cdda0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:124 +0xeb fp=0xc001487f78 sp=0xc001487ed0 pc=0x2154feb +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc0025fa0f0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc001487fc0 sp=0xc001487f78 pc=0x215759f +github.com/hashicorp/go-plugin.(*GRPCServer).Init.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x34 fp=0xc001487fe0 sp=0xc001487fc0 pc=0x215adf4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001487fe8 sp=0xc001487fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*GRPCServer).Init in goroutine 1362 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x89d + +goroutine 1364 [IO wait]: +runtime.gopark(0x2?, 0x48eaa0?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002104b30 sp=0xc002104b10 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002104b68 sp=0xc002104b30 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c4d80, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002104b88 sp=0xc002104b68 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001e55880, 0xc002788000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002104bd0 sp=0xc002104b88 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001e55860, {0xc002788000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc002104cc0 sp=0xc002104bd0 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc001f4e478, {0xc002788000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc002104d20 sp=0xc002104cc0 pc=0x51ed0d +io.(*teeReader).Read(0xc002180d40, {0xc002788000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:620 +0x5d fp=0xc002104d80 sp=0xc002104d20 pc=0x501c5d +bufio.(*Reader).Read(0xc002104f18, {0xc0025fd000, 0x400, 0x400}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc002104e40 sp=0xc002104d80 pc=0x5f943e +github.com/hashicorp/go-plugin.copyChan({0x5b611c8, 0xc0025c2ea0}, 0xc001d9d800?, {0x5b0ee40?, 0xc002180d40}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:184 +0x1f4 fp=0xc002104f80 sp=0xc002104e40 pc=0x215ccf4 +github.com/hashicorp/go-plugin.newGRPCStdioServer.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:40 +0x6c fp=0xc002104fe0 sp=0xc002104f80 pc=0x215b9ac +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002104fe8 sp=0xc002104fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCStdioServer in goroutine 1362 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:40 +0x169 + +goroutine 1365 [IO wait]: +runtime.gopark(0x10?, 0xc001fab200?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00234cb30 sp=0xc00234cb10 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00234cb68 sp=0xc00234cb30 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169d390, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00234cb88 sp=0xc00234cb68 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001e55940, 0xc002789000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00234cbd0 sp=0xc00234cb88 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001e55920, {0xc002789000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00234ccc0 sp=0xc00234cbd0 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc001f4e488, {0xc002789000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc00234cd20 sp=0xc00234ccc0 pc=0x51ed0d +io.(*teeReader).Read(0xc002180d60, {0xc002789000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:620 +0x5d fp=0xc00234cd80 sp=0xc00234cd20 pc=0x501c5d +bufio.(*Reader).Read(0xc00234cf18, {0xc0025fd400, 0x400, 0x400}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00234ce40 sp=0xc00234cd80 pc=0x5f943e +github.com/hashicorp/go-plugin.copyChan({0x5b611c8, 0xc0025c2ea0}, 0xc001d9d860?, {0x5b0ee40?, 0xc002180d60}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:184 +0x1f4 fp=0xc00234cf80 sp=0xc00234ce40 pc=0x215ccf4 +github.com/hashicorp/go-plugin.newGRPCStdioServer.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:41 +0x6c fp=0xc00234cfe0 sp=0xc00234cf80 pc=0x215b90c +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00234cfe8 sp=0xc00234cfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCStdioServer in goroutine 1362 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:41 +0x265 + +goroutine 1366 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0022dd940 sp=0xc0022dd920 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0022dd978 sp=0xc0022dd940 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c4e78, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0022dd998 sp=0xc0022dd978 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0025f2620, 0xc0022dda01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0022dd9e0 sp=0xc0022dd998 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0025f2600) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0022ddad0 sp=0xc0022dd9e0 pc=0x513f65 +net.(*netFD).accept(0xc0025f2600) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0022ddbc8 sp=0xc0022ddad0 pc=0x8967be +net.(*UnixListener).accept(0xc0025cd9e0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc0022ddc08 sp=0xc0022ddbc8 pc=0x8cedfe +net.(*UnixListener).Accept(0xc0025cd9e0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc0022ddc78 sp=0xc0022ddc08 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc002180d20) + :1 +0x44 fp=0xc0022ddcb0 sp=0xc0022ddc78 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc001853680, {0x5b2c6a0, 0xc002180d20}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc0022ddee0 sp=0xc0022ddcb0 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCServer).Serve(0xc0025c2fc0, {0x5b2c6a0, 0xc002180d20}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:153 +0xdd fp=0xc0022ddf98 sp=0xc0022ddee0 pc=0x215b33d +github.com/hashicorp/go-plugin.Serve.func6() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x62 fp=0xc0022ddfe0 sp=0xc0022ddf98 pc=0x2166b62 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0022ddfe8 sp=0xc0022ddfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.Serve in goroutine 1362 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x1f67 + +goroutine 1477 [select]: +runtime.gopark(0xc002c0ced8?, 0x2?, 0xb4?, 0x1b?, 0xc002c0ce84?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c0ccf8 sp=0xc002c0ccd8 pc=0x4777ae +runtime.selectgo(0xc002c0ced8, 0xc002c0ce80, 0xf?, 0x0, 0x8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c0ce48 sp=0xc002c0ccf8 pc=0x48916b +github.com/rancher/opni/pkg/util.ServeHandler({0x5b426e0, 0xc002c2e550}, {0x5b13c00?, 0xc002514340}, {0x5b2a6d0?, 0xc002bf8b80}) + /home/ec2-user/opni/pkg/util/servers.go:21 +0x26a fp=0xc002c0cf08 sp=0xc002c0ce48 pc=0x176a94a +github.com/rancher/opni/pkg/agent/v2.(*Agent).ListenAndServe.func2() + /home/ec2-user/opni/pkg/agent/v2/agent.go:442 +0x1b9 fp=0xc002c0cfa0 sp=0xc002c0cf08 pc=0x21bf359 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc002c0cfe0 sp=0xc002c0cfa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c0cfe8 sp=0xc002c0cfe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1320 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1368 [chan receive]: +runtime.gopark(0xc0025fa558?, 0x7c5b129634f4f?, 0x0?, 0x80?, 0xc000072780?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015e5d10 sp=0xc0015e5cf0 pc=0x4777ae +runtime.chanrecv(0xc001e55c80, 0xc0015e5e40, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0015e5d88 sp=0xc0015e5d10 pc=0x441325 +runtime.chanrecv2(0x3b9aca00?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0015e5db0 sp=0xc0015e5d88 pc=0x440f92 +github.com/hashicorp/go-plugin.pidWait(0x1860a0?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/process.go:20 +0xdb fp=0xc0015e5e68 sp=0xc0015e5db0 pc=0x216033b +github.com/hashicorp/go-plugin.(*Client).reattach.func1(0xc000b3d0e0?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:853 +0xd8 fp=0xc0015e5fb8 sp=0xc0015e5e68 pc=0x21528b8 +github.com/hashicorp/go-plugin.(*Client).reattach.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:862 +0x42 fp=0xc0015e5fe0 sp=0xc0015e5fb8 pc=0x21527a2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015e5fe8 sp=0xc0015e5fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*Client).reattach in goroutine 1367 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:846 +0x365 + +goroutine 1369 [select]: +runtime.gopark(0xc0024dbf70?, 0x2?, 0xb4?, 0x1b?, 0xc0024dbebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024dbd20 sp=0xc0024dbd00 pc=0x4777ae +runtime.selectgo(0xc0024dbf70, 0xc0024dbeb8, 0xc0027a2a00?, 0x0, 0xc0027a2b00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0024dbe70 sp=0xc0024dbd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002181200, {0x5b426e0, 0xc0025fa370}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0024dbfa0 sp=0xc0024dbe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0024dbfe0 sp=0xc0024dbfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024dbfe8 sp=0xc0024dbfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1367 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1370 [select]: +runtime.gopark(0xc0027c9f70?, 0x2?, 0xb4?, 0x1b?, 0xc0027c9ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0027c9d20 sp=0xc0027c9d00 pc=0x4777ae +runtime.selectgo(0xc0027c9f70, 0xc0027c9eb8, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0027c9e70 sp=0xc0027c9d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002181260, {0x5b426e0, 0xc0025fa3c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0027c9fa0 sp=0xc0027c9e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0027c9fe0 sp=0xc0027c9fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0027c9fe8 sp=0xc0027c9fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1367 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1399 [select]: +runtime.gopark(0xc0029b5fb0?, 0x2?, 0xb4?, 0x1b?, 0xc0029b5f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029b5de8 sp=0xc0029b5dc8 pc=0x4777ae +runtime.selectgo(0xc0029b5fb0, 0xc0029b5f70, 0xc0029b5fd0?, 0x0, 0xc002715570?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029b5f38 sp=0xc0029b5de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0029b5fe0 sp=0xc0029b5f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029b5fe8 sp=0xc0029b5fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1376 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1400 [select]: +runtime.gopark(0xc0015e6fb0?, 0x2?, 0xb0?, 0x6d?, 0xc0015e6ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015e6d58 sp=0xc0015e6d38 pc=0x4777ae +runtime.selectgo(0xc0015e6fb0, 0xc0015e6ee0, 0xc0015e6f98?, 0x0, 0xc002780600?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0015e6ea8 sp=0xc0015e6d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0015e6fe0 sp=0xc0015e6ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015e6fe8 sp=0xc0015e6fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1376 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1420 [IO wait]: +runtime.gopark(0xc0027c4fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0027c4f98 sp=0xc0027c4f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0027c4fd0 sp=0xc0027c4f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c4a98, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0027c4ff0 sp=0xc0027c4fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0025f2ba0, 0xc00284a000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0027c5038 sp=0xc0027c4ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0025f2b80, {0xc00284a000, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0027c5128 sp=0xc0027c5038 pc=0x50e5a5 +net.(*netFD).Read(0xc0025f2b80, {0xc00284a000, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0027c5188 sp=0xc0027c5128 pc=0x89342b +net.(*conn).Read(0xc001f4e568, {0xc00284a000, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0027c5210 sp=0xc0027c5188 pc=0x8b176d +net.(*UnixConn).Read(0xc001f4e568, {0xc00284a000, 0x600, 0x600}) + :1 +0x54 fp=0xc0027c5258 sp=0xc0027c5210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0030c4510, {0xc00284a000, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0027c52b8 sp=0xc0027c5258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc00125fea8, {0x5b0f6c0, 0xc0030c4510}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0027c5320 sp=0xc0027c52b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc00125fc00, {0x5b0f840?, 0xc001f4e568}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc0027c5390 sp=0xc0027c5320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc00125fc00, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0027c59c0 sp=0xc0027c5390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc00125fc00, {0xc002bb4000, 0x8000, 0xc0027c5af8?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc0027c5aa8 sp=0xc0027c59c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001d873c0, {0xc002bb4000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc0027c5af0 sp=0xc0027c5aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc00245eae0, {0xc0014f91c0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0027c5bb0 sp=0xc0027c5af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00245eae0}, {0xc0014f91c0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0027c5bf8 sp=0xc0027c5bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0014f91c0, 0x9, 0x9}, {0x5b0f0a0, 0xc00245eae0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0027c5c98 sp=0xc0027c5bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0014f9180) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0027c5dd0 sp=0xc0027c5c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001d846c0, 0xc000813330?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0027c5fb0 sp=0xc0027c5dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0027c5fe0 sp=0xc0027c5fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0027c5fe8 sp=0xc0027c5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1372 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 1375 [select]: +runtime.gopark(0xc001bc3f48?, 0x2?, 0x2a?, 0xce?, 0xc001bc3f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bc3d80 sp=0xc001bc3d60 pc=0x4777ae +runtime.selectgo(0xc001bc3f48, 0xc001bc3f00, 0xc001bc3f08?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001bc3ed0 sp=0xc001bc3d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).Recv(0xc001ab9c40) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:234 +0xeb fp=0xc001bc3f78 sp=0xc001bc3ed0 pc=0x2155d2b +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc0025fa8c0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc001bc3fc0 sp=0xc001bc3f78 pc=0x215759f +github.com/hashicorp/go-plugin.newGRPCClient.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x34 fp=0xc001bc3fe0 sp=0xc001bc3fc0 pc=0x21599b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bc3fe8 sp=0xc001bc3fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 1367 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x5b7 + +goroutine 1376 [select]: +runtime.gopark(0xc0022f7748?, 0x2?, 0xd0?, 0x75?, 0xc0022f76fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0022f7578 sp=0xc0022f7558 pc=0x4777ae +runtime.selectgo(0xc0022f7748, 0xc0022f76f8, 0xc0020467c0?, 0x0, 0xc0025f2f18?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0022f76c8 sp=0xc0022f7578 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001d56ea0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc0022f7778 sp=0xc0022f76c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc002782d00, {0x43dd860?, 0xc0026a5400}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0022f79d8 sp=0xc0022f7778 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0019cad14?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc0022f7a38 sp=0xc0022f79d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc0019cac60, 0xc0022f7bf0, 0xc0022f7bc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc0022f7ae0 sp=0xc0022f7a38 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019cac60, {0x43dd860, 0xc0026a5400}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc0022f7c20 sp=0xc0022f7ae0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc001ff1e00, {0x43dd860, 0xc0026a5400}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc0022f7de8 sp=0xc0022f7c20 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamClient).Recv(0xc0026882c0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:132 +0x66 fp=0xc0022f7e28 sp=0xc0022f7de8 pc=0x2102606 +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream(0xc001ab9c40) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:197 +0x2e8 fp=0xc0022f7fc0 sp=0xc0022f7e28 pc=0x2155468 +github.com/hashicorp/go-plugin.newGRPCClient.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x34 fp=0xc0022f7fe0 sp=0xc0022f7fc0 pc=0x2159954 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0022f7fe8 sp=0xc0022f7fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 1367 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x62e + +goroutine 1401 [chan receive]: +runtime.gopark(0x40a16a0?, 0x0?, 0x10?, 0x81?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002b9fd60 sp=0xc002b9fd40 pc=0x4777ae +runtime.chanrecv(0xc001be5d40, 0xc002b9fe60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002b9fdd8 sp=0xc002b9fd60 pc=0x441325 +runtime.chanrecv1(0xc0026a5208?, 0x40a16a0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002b9fe00 sp=0xc002b9fdd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002b9ffe0 sp=0xc002b9fe00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002b9ffe8 sp=0xc002b9ffe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1376 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1391 [select]: +runtime.gopark(0xc0024c9790?, 0x2?, 0x18?, 0x96?, 0xc0024c9744?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024c95c0 sp=0xc0024c95a0 pc=0x4777ae +runtime.selectgo(0xc0024c9790, 0xc0024c9740, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0024c9710 sp=0xc0024c95c0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c6ac60) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc0024c97c0 sp=0xc0024c9710 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc0026a8820, {0x43c4ea0?, 0xc0021e6240}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0024c9a20 sp=0xc0024c97c0 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001d56894?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc0024c9a80 sp=0xc0024c9a20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001d567e0, 0xc00234ac38, 0xc00234ac08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc0024c9b28 sp=0xc0024c9a80 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d567e0, {0x43c4ea0, 0xc0021e6240}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc0024c9c68 sp=0xc0024c9b28 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc001e41f00, {0x43c4ea0, 0xc0021e6240}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc0024c9e30 sp=0xc0024c9c68 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCStdioStreamStdioClient).Recv(0xc002577000) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:160 +0x66 fp=0xc0024c9e70 sp=0xc0024c9e30 pc=0x2103f26 +github.com/hashicorp/go-plugin.(*grpcStdioClient).Run(0xc001d86f80, {0x5b0ee60, 0x779c980}, {0x5b0eb80, 0xc00007c038}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:135 +0x11c fp=0xc0024c9f80 sp=0xc0024c9e70 pc=0x215c15c +github.com/hashicorp/go-plugin.newGRPCClient.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x6c fp=0xc0024c9fe0 sp=0xc0024c9f80 pc=0x21598ec +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024c9fe8 sp=0xc0024c9fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 1361 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x84f + +goroutine 1392 [select]: +runtime.gopark(0xc00234afb0?, 0x2?, 0xb4?, 0x1b?, 0xc00234af74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00234ade8 sp=0xc00234adc8 pc=0x4777ae +runtime.selectgo(0xc00234afb0, 0xc00234af70, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00234af38 sp=0xc00234ade8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc00234afe0 sp=0xc00234af38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00234afe8 sp=0xc00234afe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1361 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1393 [select]: +runtime.gopark(0xc0029b7fb0?, 0x2?, 0xb0?, 0x7d?, 0xc0029b7ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029b7d58 sp=0xc0029b7d38 pc=0x4777ae +runtime.selectgo(0xc0029b7fb0, 0xc0029b7ee0, 0x5655434a55573374?, 0x0, 0x6a55575255575268?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029b7ea8 sp=0xc0029b7d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0029b7fe0 sp=0xc0029b7ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029b7fe8 sp=0xc0029b7fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1410 [chan receive]: +runtime.gopark(0xc0014f90c0?, 0xc001822eb8?, 0xbb?, 0xe2?, 0xc0014f90a0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001822d60 sp=0xc001822d40 pc=0x4777ae +runtime.chanrecv(0xc0022fb500, 0xc001822e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001822dd8 sp=0xc001822d60 pc=0x441325 +runtime.chanrecv1(0xc00146bf60?, 0x10100c001423608?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001822e00 sp=0xc001822dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001822fe0 sp=0xc001822e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001822fe8 sp=0xc001822fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1411 [select]: +runtime.gopark(0xc002608fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002608f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002608de8 sp=0xc002608dc8 pc=0x4777ae +runtime.selectgo(0xc002608fb0, 0xc002608f70, 0xc001d9d320?, 0x0, 0xc001d9d380?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002608f38 sp=0xc002608de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002608fe0 sp=0xc002608f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002608fe8 sp=0xc002608fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1294 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1412 [select]: +runtime.gopark(0xc002609fb0?, 0x2?, 0xb0?, 0x9d?, 0xc002609ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002609d58 sp=0xc002609d38 pc=0x4777ae +runtime.selectgo(0xc002609fb0, 0xc002609ee0, 0x203a737365726464?, 0x0, 0xa31313635343a74?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002609ea8 sp=0xc002609d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002609fe0 sp=0xc002609ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002609fe8 sp=0xc002609fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1294 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1413 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002162d60 sp=0xc002162d40 pc=0x4777ae +runtime.chanrecv(0xc0022fb740, 0xc002162e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002162dd8 sp=0xc002162d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002162e00 sp=0xc002162dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002162fe0 sp=0xc002162e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002162fe8 sp=0xc002162fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1294 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1414 [select]: +runtime.gopark(0xc00260afa0?, 0x3?, 0x0?, 0x0?, 0xc00260af3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00260ada8 sp=0xc00260ad88 pc=0x4777ae +runtime.selectgo(0xc00260afa0, 0xc00260af34, 0xc00260af40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00260aef8 sp=0xc00260ada8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:184 +0x1b3 fp=0xc00260afe0 sp=0xc00260aef8 pc=0x21557f3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00260afe8 sp=0xc00260afe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream in goroutine 1294 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:182 +0x24a + +goroutine 1415 [select]: +runtime.gopark(0xc002b9b310?, 0x3?, 0xd9?, 0xea?, 0xc002b9b292?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002b9b0f0 sp=0xc002b9b0d0 pc=0x4777ae +runtime.selectgo(0xc002b9b310, 0xc002b9b28c, 0x46216a2?, 0x0, 0x4?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002b9b240 sp=0xc002b9b0f0 pc=0x48916b +github.com/hashicorp/go-plugin.(*grpcStdioServer).StreamStdio(0xc00234fc60, 0x42219c0?, {0x5b4ebb0, 0xc002577440}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:61 +0x1e5 fp=0xc002b9b350 sp=0xc002b9b240 pc=0x215bbc5 +github.com/hashicorp/go-plugin/internal/plugin._GRPCStdio_StreamStdio_Handler({0x409e580?, 0xc00234fc60}, {0x5b4a6f0, 0xc001d871e0}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:185 +0xf7 fp=0xc002b9b3b0 sp=0xc002b9b350 pc=0x2104077 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x409e580, 0xc00234fc60}, {0x5b4ab30, 0xc002711ec0}, 0xc000127980?, 0x47efca8) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002b9b400 sp=0xc002b9b3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x409e580, 0xc00234fc60}, {0x5b4ab30, 0xc002711ec0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002b9b490 sp=0xc002b9b400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x409e580, 0xc00234fc60}, {0x5b492f8, 0xc001389590}, 0xc000127980, 0xc0021e6500) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002b9b7d0 sp=0xc002b9b490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x409e580, 0xc00234fc60}, {0x5b492f8, 0xc001389590}, 0x3f252a0?, 0xc002577370?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002b9b848 sp=0xc002b9b7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c6b320, 0xc0025cd470, 0x770cba0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002b9bdb0 sp=0xc002b9b848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c6b320, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002b9bf28 sp=0xc002b9bdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002b9bfe0 sp=0xc002b9bf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002b9bfe8 sp=0xc002b9bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1387 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1416 [select]: +runtime.gopark(0xc0025e6890?, 0x2?, 0x98?, 0x67?, 0xc0025e683c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0025e66a8 sp=0xc0025e6688 pc=0x4777ae +runtime.selectgo(0xc0025e6890, 0xc0025e6838, 0x7f429a489a00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0025e67f8 sp=0xc0025e66a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0028462d0, {0xc002711fc0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0025e68c0 sp=0xc0025e67f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0028462d0, {0xc002711fc0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0025e6958 sp=0xc0025e68c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002711b60, {0xc002711fc0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0025e69a0 sp=0xc0025e6958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002711b60}, {0xc002711fc0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0025e69e8 sp=0xc0025e69a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c6b560, {0xc002711fc0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0025e6a58 sp=0xc0025e69e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002711fb0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0025e6b28 sp=0xc0025e6a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0021e6e80?, 0xc001c6b560, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0025e6c18 sp=0xc0025e6b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002711fb0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc002846780}, 0xc0025ccf60?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0025e6cd0 sp=0xc0025e6c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001389680, {0x43dcf60?, 0xc002846780}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0025e6f08 sp=0xc0025e6cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002848120, {0x43dcf60, 0xc002846780}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0025e6ff0 sp=0xc0025e6f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc001d87240, {0x43dcf60, 0xc002846780}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0025e7028 sp=0xc0025e6ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc002577550) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc0025e7068 sp=0xc0025e7028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc002577550?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0025e70a8 sp=0xc0025e7068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc002577560) + :1 +0x46 fp=0xc0025e70e0 sp=0xc0025e70a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0025cd0e0, {0x5b51368, 0xc002577560}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc0025e7320 sp=0xc0025e70e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00234fbc0, {0x5b51038, 0xc002577550}) + :1 +0xaf fp=0xc0025e7370 sp=0xc0025e7320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00234fbc0}, {0x5b4a6f0?, 0xc001d87240}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0025e73b0 sp=0xc0025e7370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00234fbc0}, {0x5b4ab30, 0xc002848120}, 0xc000127b48?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0025e7400 sp=0xc0025e73b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00234fbc0}, {0x5b4ab30, 0xc002848120}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0025e7490 sp=0xc0025e7400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00234fbc0}, {0x5b492f8, 0xc001389680}, 0xc000127b48, 0xc0021e6700) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0025e77d0 sp=0xc0025e7490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00234fbc0}, {0x5b492f8, 0xc001389680}, 0x3f252a0?, 0xc002577480?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0025e7848 sp=0xc0025e77d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c6b560, 0xc0025cd170, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0025e7db0 sp=0xc0025e7848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c6b560, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0025e7f28 sp=0xc0025e7db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0025e7fe0 sp=0xc0025e7f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0025e7fe8 sp=0xc0025e7fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1387 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1417 [select]: +runtime.gopark(0xc0027c2a68?, 0x2?, 0x40?, 0xa5?, 0xc0027c2a14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0027c2880 sp=0xc0027c2860 pc=0x4777ae +runtime.selectgo(0xc0027c2a68, 0xc0027c2a10, 0x7f429a489700?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0027c29d0 sp=0xc0027c2880 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0028463c0, {0xc0028482b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0027c2a98 sp=0xc0027c29d0 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0028463c0, {0xc0028482b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0027c2b30 sp=0xc0027c2a98 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002711ce0, {0xc0028482b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0027c2b78 sp=0xc0027c2b30 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002711ce0}, {0xc0028482b0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0027c2bc0 sp=0xc0027c2b78 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c6b680, {0xc0028482b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0027c2c30 sp=0xc0027c2bc0 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0028482a0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0027c2d00 sp=0xc0027c2c30 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0027c1570?, 0xc001c6b680, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0027c2df0 sp=0xc0027c2d00 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0028482a0, {0x7f424347ae80, 0x779c980}, 0xc001389680?, {0x0, 0x0}, {0x43dd860, 0xc002846870}, 0x12b01b3?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0027c2ea8 sp=0xc0027c2df0 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001389770, {0x43dd860?, 0xc002846870}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0027c30e0 sp=0xc0027c2ea8 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0028483f0, {0x43dd860, 0xc002846870}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0027c31c8 sp=0xc0027c30e0 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc001d872c0, {0x43dd860, 0xc002846870}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0027c3200 sp=0xc0027c31c8 pc=0x1fd94b2 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamServer).Recv(0xc002577690) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:167 +0x66 fp=0xc0027c3240 sp=0xc0027c3200 pc=0x2102886 +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream(0xc0025cd230, {0x5b512b8, 0xc002577690}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:87 +0x218 fp=0xc0027c3370 sp=0xc0027c3240 pc=0x2154718 +github.com/hashicorp/go-plugin/internal/plugin._GRPCBroker_StartStream_Handler({0x41cb860?, 0xc0025cd230}, {0x5b4a6f0?, 0xc001d872c0}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:148 +0xb6 fp=0xc0027c33b0 sp=0xc0027c3370 pc=0x2102716 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41cb860, 0xc0025cd230}, {0x5b4ab30, 0xc0028483f0}, 0xc000127f08?, 0x47efc98) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0027c3400 sp=0xc0027c33b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41cb860, 0xc0025cd230}, {0x5b4ab30, 0xc0028483f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0027c3490 sp=0xc0027c3400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41cb860, 0xc0025cd230}, {0x5b492f8, 0xc001389770}, 0xc000127f08, 0xc0021e6ec0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0027c37d0 sp=0xc0027c3490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41cb860, 0xc0025cd230}, {0x5b492f8, 0xc001389770}, 0x3f252a0?, 0xc0025775c0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0027c3848 sp=0xc0027c37d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c6b680, 0xc0025cd2c0, 0x770cb80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0027c3db0 sp=0xc0027c3848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c6b680, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0027c3f28 sp=0xc0027c3db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0027c3fe0 sp=0xc0027c3f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0027c3fe8 sp=0xc0027c3fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1387 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1418 [select]: +runtime.gopark(0xc0029b3fa0?, 0x3?, 0x0?, 0x4e?, 0xc0029b3f3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029b3da8 sp=0xc0029b3d88 pc=0x4777ae +runtime.selectgo(0xc0029b3fa0, 0xc0029b3f34, 0xc0029b3f40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029b3ef8 sp=0xc0029b3da8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:73 +0x1ad fp=0xc0029b3fe0 sp=0xc0029b3ef8 pc=0x2154aad +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029b3fe8 sp=0xc0029b3fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream in goroutine 1417 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:71 +0x1d1 + +goroutine 1432 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00260d9e0 sp=0xc00260d9c0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00260da18 sp=0xc00260d9e0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c45c0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00260da38 sp=0xc00260da18 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc002c192a0, 0x448801?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00260da80 sp=0xc00260da38 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc002c19280) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc00260db70 sp=0xc00260da80 pc=0x513f65 +net.(*netFD).accept(0xc002c19280) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc00260dc68 sp=0xc00260db70 pc=0x8967be +net.(*TCPListener).accept(0xc002bf8b80) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc00260dcb0 sp=0xc00260dc68 pc=0x8c383e +net.(*TCPListener).Accept(0xc002bf8b80) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc00260dd20 sp=0xc00260dcb0 pc=0x8c1ba5 +net/http.(*onceCloseListener).Accept(0xc002781ec0) + :1 +0x44 fp=0xc00260dd58 sp=0xc00260dd20 pc=0xab5544 +net/http.(*Server).Serve(0xc00278a1e0, {0x5b2a6d0, 0xc002bf8b80}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc00260df50 sp=0xc00260dd58 pc=0xa7859f +github.com/rancher/opni/pkg/util.ServeHandler.func1() + /home/ec2-user/opni/pkg/util/servers.go:19 +0x47 fp=0xc00260dfa0 sp=0xc00260df50 pc=0x176aa67 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc00260dfe0 sp=0xc00260dfa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00260dfe8 sp=0xc00260dfe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1477 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1426 [select]: +runtime.gopark(0xc00260cfb0?, 0x2?, 0xb4?, 0x1b?, 0xc00260cf74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00260cde8 sp=0xc00260cdc8 pc=0x4777ae +runtime.selectgo(0xc00260cfb0, 0xc00260cf70, 0xc00235c9c0?, 0x0, 0xc00235ca20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00260cf38 sp=0xc00260cde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc00260cfe0 sp=0xc00260cf38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00260cfe8 sp=0xc00260cfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1361 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1427 [select]: +runtime.gopark(0xc002106fb0?, 0x2?, 0xb0?, 0x6d?, 0xc002106ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002106d58 sp=0xc002106d38 pc=0x4777ae +runtime.selectgo(0xc002106fb0, 0xc002106ee0, 0xc00073add1?, 0x0, 0x5b058a8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002106ea8 sp=0xc002106d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002106fe0 sp=0xc002106ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002106fe8 sp=0xc002106fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1428 [chan receive]: +runtime.gopark(0xc002405da8?, 0x1103325?, 0x0?, 0xa7?, 0xc0025253a8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002405d60 sp=0xc002405d40 pc=0x4777ae +runtime.chanrecv(0xc00235cea0, 0xc002405e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002405dd8 sp=0xc002405d60 pc=0x441325 +runtime.chanrecv1(0xc0026525d0?, 0xc000f1f560?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002405e00 sp=0xc002405dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002405fe0 sp=0xc002405e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002405fe8 sp=0xc002405fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1435 [select]: +runtime.gopark(0xc002d87fb0?, 0x2?, 0x0?, 0x0?, 0xc002d87f2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d87da8 sp=0xc002d87d88 pc=0x4777ae +runtime.selectgo(0xc002d87fb0, 0xc002d87f28, 0x0?, 0x0, 0x4ab492?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d87ef8 sp=0xc002d87da8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002d87fe0 sp=0xc002d87ef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d87fe8 sp=0xc002d87fe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1419 [select]: +runtime.gopark(0xc002774890?, 0x2?, 0x4e?, 0x1a?, 0xc00277483c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0027746a8 sp=0xc002774688 pc=0x4777ae +runtime.selectgo(0xc002774890, 0xc002774838, 0x7f429a488200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0027747f8 sp=0xc0027746a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002846960, {0xc002780fd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0027748c0 sp=0xc0027747f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002846960, {0xc002780fd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002774958 sp=0xc0027748c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0028485d0, {0xc002780fd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0027749a0 sp=0xc002774958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0028485d0}, {0xc002780fd0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0027749e8 sp=0xc0027749a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c6b7a0, {0xc002780fd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002774a58 sp=0xc0027749e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002780fc0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002774b28 sp=0xc002774a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc00223ad00?, 0xc001c6b7a0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002774c18 sp=0xc002774b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002780fc0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0025fb450}, 0xc0025ccf60?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002774cd0 sp=0xc002774c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc00278a000, {0x43dcf60?, 0xc0025fb450}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002774f08 sp=0xc002774cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002781110, {0x43dcf60, 0xc0025fb450}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002774ff0 sp=0xc002774f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002181800, {0x43dcf60, 0xc0025fb450}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002775028 sp=0xc002774ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc002856c30) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc002775068 sp=0xc002775028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc002856c30?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0027750a8 sp=0xc002775068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc002856c40) + :1 +0x46 fp=0xc0027750e0 sp=0xc0027750a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0025cd0e0, {0x5b51368, 0xc002856c40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc002775320 sp=0xc0027750e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00234fbc0, {0x5b51038, 0xc002856c30}) + :1 +0xaf fp=0xc002775370 sp=0xc002775320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00234fbc0}, {0x5b4a6f0?, 0xc002181800}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0027753b0 sp=0xc002775370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00234fbc0}, {0x5b4ab30, 0xc002781110}, 0xc00149eac8?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002775400 sp=0xc0027753b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00234fbc0}, {0x5b4ab30, 0xc002781110}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002775490 sp=0xc002775400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00234fbc0}, {0x5b492f8, 0xc00278a000}, 0xc00149eac8, 0xc00223a600) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0027757d0 sp=0xc002775490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00234fbc0}, {0x5b492f8, 0xc00278a000}, 0x3f252a0?, 0xc002856b60?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002775848 sp=0xc0027757d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c6b7a0, 0xc0025cd170, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002775db0 sp=0xc002775848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c6b7a0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002775f28 sp=0xc002775db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002775fe0 sp=0xc002775f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002775fe8 sp=0xc002775fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1387 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1448 [chan receive]: +runtime.gopark(0xc0007defc0?, 0xc0007df0a0?, 0xf0?, 0xf1?, 0xc0007df2d0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029b0d60 sp=0xc0029b0d40 pc=0x4777ae +runtime.chanrecv(0xc001f03a40, 0xc0029b0e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0029b0dd8 sp=0xc0029b0d60 pc=0x441325 +runtime.chanrecv1(0xc000ee7320?, 0xc00256ec80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0029b0e00 sp=0xc0029b0dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc0029b0fe0 sp=0xc0029b0e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029b0fe8 sp=0xc0029b0fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1455 [select]: +runtime.gopark(0xc002102fb0?, 0x2?, 0x0?, 0x4e?, 0xc002102f2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002102da8 sp=0xc002102d88 pc=0x4777ae +runtime.selectgo(0xc002102fb0, 0xc002102f28, 0x4b0571?, 0x0, 0xc002102fd0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002102ef8 sp=0xc002102da8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002102fe0 sp=0xc002102ef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002102fe8 sp=0xc002102fe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1454 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1450 [select]: +runtime.gopark(0xc002ba6890?, 0x2?, 0x98?, 0x67?, 0xc002ba683c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ba66a8 sp=0xc002ba6688 pc=0x4777ae +runtime.selectgo(0xc002ba6890, 0xc002ba6838, 0x7f429a486a00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ba67f8 sp=0xc002ba66a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002658fa0, {0xc0026538a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002ba68c0 sp=0xc002ba67f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002658fa0, {0xc0026538a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002ba6958 sp=0xc002ba68c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002653680, {0xc0026538a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002ba69a0 sp=0xc002ba6958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002653680}, {0xc0026538a0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002ba69e8 sp=0xc002ba69a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0018f58c0, {0xc0026538a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002ba6a58 sp=0xc002ba69e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002653890, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002ba6b28 sp=0xc002ba6a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001d7c780?, 0xc0018f58c0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002ba6c18 sp=0xc002ba6b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002653890, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0026593b0}, 0xc0025ccf60?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002ba6cd0 sp=0xc002ba6c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000b9c1e0, {0x43dcf60?, 0xc0026593b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002ba6f08 sp=0xc002ba6cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0026539e0, {0x43dcf60, 0xc0026593b0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002ba6ff0 sp=0xc002ba6f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002488160, {0x43dcf60, 0xc0026593b0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002ba7028 sp=0xc002ba6ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00271e780) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc002ba7068 sp=0xc002ba7028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00271e780?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc002ba70a8 sp=0xc002ba7068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00271e790) + :1 +0x46 fp=0xc002ba70e0 sp=0xc002ba70a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0025cd0e0, {0x5b51368, 0xc00271e790}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc002ba7320 sp=0xc002ba70e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00234fbc0, {0x5b51038, 0xc00271e780}) + :1 +0xaf fp=0xc002ba7370 sp=0xc002ba7320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00234fbc0}, {0x5b4a6f0?, 0xc002488160}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc002ba73b0 sp=0xc002ba7370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00234fbc0}, {0x5b4ab30, 0xc0026539e0}, 0xc0005817d0?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002ba7400 sp=0xc002ba73b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00234fbc0}, {0x5b4ab30, 0xc0026539e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002ba7490 sp=0xc002ba7400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00234fbc0}, {0x5b492f8, 0xc000b9c1e0}, 0xc0005817d0, 0xc002171f80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002ba77d0 sp=0xc002ba7490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00234fbc0}, {0x5b492f8, 0xc000b9c1e0}, 0x3f252a0?, 0xc00271e6b0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002ba7848 sp=0xc002ba77d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc0018f58c0, 0xc0025cd170, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002ba7db0 sp=0xc002ba7848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc0018f58c0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002ba7f28 sp=0xc002ba7db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002ba7fe0 sp=0xc002ba7f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ba7fe8 sp=0xc002ba7fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1387 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1456 [select]: +runtime.gopark(0xc002c84f90?, 0x2?, 0xff?, 0xff?, 0xc002c84eac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c84d28 sp=0xc002c84d08 pc=0x4777ae +runtime.selectgo(0xc002c84f90, 0xc002c84ea8, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c84e78 sp=0xc002c84d28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc0015ec240) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002c84fc0 sp=0xc002c84e78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002c84fe0 sp=0xc002c84fc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c84fe8 sp=0xc002c84fe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1454 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1452 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002107930 sp=0xc002107910 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002107968 sp=0xc002107930 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c46b8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002107988 sp=0xc002107968 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00182d720, 0x448801?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0021079d0 sp=0xc002107988 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc00182d700) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc002107ac0 sp=0xc0021079d0 pc=0x513f65 +net.(*netFD).accept(0xc00182d700) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc002107bb8 sp=0xc002107ac0 pc=0x8967be +net.(*TCPListener).accept(0xc002488200) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc002107c00 sp=0xc002107bb8 pc=0x8c383e +net.(*TCPListener).Accept(0xc002488200) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc002107c70 sp=0xc002107c00 pc=0x8c1ba5 +net/http.(*onceCloseListener).Accept(0xc002781380) + :1 +0x44 fp=0xc002107ca8 sp=0xc002107c70 pc=0xab5544 +net/http.(*Server).Serve(0xc00278a0f0, {0x5b2a6d0, 0xc002488200}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc002107ea0 sp=0xc002107ca8 pc=0xa7859f +net/http.Serve(...) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:2595 +github.com/gin-gonic/gin.(*Engine).RunListener(0xc0025eeea0, {0x5b2a6d0, 0xc002488200}) + /home/ec2-user/go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/gin.go:565 +0x370 fp=0xc002107f90 sp=0xc002107ea0 pc=0x171e870 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure.func1() + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:72 +0x5c fp=0xc002107fe0 sp=0xc002107f90 pc=0x262a57c +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002107fe8 sp=0xc002107fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure in goroutine 1451 + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:71 +0x26d + +goroutine 1421 [select]: +runtime.gopark(0xc002bf3e70?, 0x2?, 0x0?, 0x0?, 0xc002bf3df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002bf3c58 sp=0xc002bf3c38 pc=0x4777ae +runtime.selectgo(0xc002bf3e70, 0xc002bf3df0, 0xc002bf3e00?, 0x0, 0xc00125fc00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002bf3da8 sp=0xc002bf3c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002846a50, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002bf3ea0 sp=0xc002bf3da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002715570) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002bf3f58 sp=0xc002bf3ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc002bf3fe0 sp=0xc002bf3f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002bf3fe8 sp=0xc002bf3fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1372 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1396 [select]: +runtime.gopark(0xc002bebe48?, 0x2?, 0x0?, 0x0?, 0xc002bebdcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002bebc30 sp=0xc002bebc10 pc=0x4777ae +runtime.selectgo(0xc002bebe48, 0xc002bebdc8, 0xc002bebdd8?, 0x0, 0xc00133b500?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002bebd80 sp=0xc002bebc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0026a51d0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002bebe78 sp=0xc002bebd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0026ae700) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002bebf30 sp=0xc002bebe78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc002bebfe0 sp=0xc002bebf30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002bebfe8 sp=0xc002bebfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1395 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1397 [select]: +runtime.gopark(0xc00260ef70?, 0x4?, 0x40?, 0x0?, 0xc00260edd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00260ec00 sp=0xc00260ebe0 pc=0x4777ae +runtime.selectgo(0xc00260ef70, 0xc00260edd0, 0xc00260ef08?, 0x0, 0x5b50310?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00260ed50 sp=0xc00260ec00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002515860) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc00260efc0 sp=0xc00260ed50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc00260efe0 sp=0xc00260efc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00260efe8 sp=0xc00260efe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1395 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1398 [IO wait]: +runtime.gopark(0xc001730f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001730f40 sp=0xc001730f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001730f78 sp=0xc001730f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c47b0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001730f98 sp=0xc001730f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00257daa0, 0xc002c6ea00?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001730fe0 sp=0xc001730f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00257da80, {0xc002c6ea00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0017310d0 sp=0xc001730fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc00257da80, {0xc002c6ea00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001731130 sp=0xc0017310d0 pc=0x89342b +net.(*conn).Read(0xc001d6f078, {0xc002c6ea00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0017311b8 sp=0xc001731130 pc=0x8b176d +net.(*UnixConn).Read(0xc001d6f078, {0xc002c6ea00, 0x500, 0x500}) + :1 +0x54 fp=0xc001731200 sp=0xc0017311b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0030c4528, {0xc002c6ea00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001731260 sp=0xc001731200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc00133b7a8, {0x5b0f6c0, 0xc0030c4528}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0017312c8 sp=0xc001731260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc00133b500, {0x5b0f840?, 0xc001d6f078}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001731338 sp=0xc0017312c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc00133b500, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001731968 sp=0xc001731338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc00133b500, {0xc002bcc000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001731a50 sp=0xc001731968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0026e0640, {0xc002bcc000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001731a98 sp=0xc001731a50 pc=0x10969ed +bufio.(*Reader).Read(0xc001de70e0, {0xc002686660, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001731b58 sp=0xc001731a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001de70e0}, {0xc002686660, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001731ba0 sp=0xc001731b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc002686660, 0x9, 0x9}, {0x5b0f0a0, 0xc001de70e0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001731c40 sp=0xc001731ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc002686620) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001731d78 sp=0xc001731c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002515860, 0x2020200a3936202d?, 0x200a313131202d20?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001731ec8 sp=0xc001731d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc001853680, {0x5b54e00?, 0xc002515860}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001731f80 sp=0xc001731ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001731fe0 sp=0xc001731f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001731fe8 sp=0xc001731fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1395 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1402 [select]: +runtime.gopark(0xc002ba0fa0?, 0x3?, 0x0?, 0x0?, 0xc002ba0f3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ba0da8 sp=0xc002ba0d88 pc=0x4777ae +runtime.selectgo(0xc002ba0fa0, 0xc002ba0f34, 0xc002ba0f40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ba0ef8 sp=0xc002ba0da8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:184 +0x1b3 fp=0xc002ba0fe0 sp=0xc002ba0ef8 pc=0x21557f3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ba0fe8 sp=0xc002ba0fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream in goroutine 1376 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:182 +0x24a + +goroutine 1403 [select]: +runtime.gopark(0xc002ba1fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002ba1f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ba1de8 sp=0xc002ba1dc8 pc=0x4777ae +runtime.selectgo(0xc002ba1fb0, 0xc002ba1f70, 0x21557a5?, 0x0, 0xc0026882c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ba1f38 sp=0xc002ba1de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002ba1fe0 sp=0xc002ba1f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ba1fe8 sp=0xc002ba1fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1367 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1404 [select]: +runtime.gopark(0xc002c0ffb0?, 0x2?, 0xb0?, 0xfd?, 0xc002c0fee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c0fd58 sp=0xc002c0fd38 pc=0x4777ae +runtime.selectgo(0xc002c0ffb0, 0xc002c0fee0, 0x1128d01?, 0x0, 0x1102359?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c0fea8 sp=0xc002c0fd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002c0ffe0 sp=0xc002c0fea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c0ffe8 sp=0xc002c0ffe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1367 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1405 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ba2d60 sp=0xc002ba2d40 pc=0x4777ae +runtime.chanrecv(0xc002bf6120, 0xc002ba2e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002ba2dd8 sp=0xc002ba2d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002ba2e00 sp=0xc002ba2dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002ba2fe0 sp=0xc002ba2e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ba2fe8 sp=0xc002ba2fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1367 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1406 [select]: +runtime.gopark(0xc0027c7790?, 0x2?, 0x18?, 0x76?, 0xc0027c7744?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0027c75c0 sp=0xc0027c75a0 pc=0x4777ae +runtime.selectgo(0xc0027c7790, 0xc0027c7740, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0027c7710 sp=0xc0027c75c0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001d570e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc0027c77c0 sp=0xc0027c7710 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc002782c30, {0x43c4ea0?, 0xc002452140}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0027c7a20 sp=0xc0027c77c0 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0019cabf4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc0027c7a80 sp=0xc0027c7a20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc0019cab40, 0xc002ba1c38, 0xc002ba1c08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc0027c7b28 sp=0xc0027c7a80 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019cab40, {0x43c4ea0, 0xc002452140}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc0027c7c68 sp=0xc0027c7b28 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc001ff1e80, {0x43c4ea0, 0xc002452140}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc0027c7e30 sp=0xc0027c7c68 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCStdioStreamStdioClient).Recv(0xc002688340) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:160 +0x66 fp=0xc0027c7e70 sp=0xc0027c7e30 pc=0x2103f26 +github.com/hashicorp/go-plugin.(*grpcStdioClient).Run(0xc0026e0a60, {0x5b0ee60, 0x779c980}, {0x5b0eb80, 0xc00007c038}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:135 +0x11c fp=0xc0027c7f80 sp=0xc0027c7e70 pc=0x215c15c +github.com/hashicorp/go-plugin.newGRPCClient.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x6c fp=0xc0027c7fe0 sp=0xc0027c7f80 pc=0x21598ec +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0027c7fe8 sp=0xc0027c7fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 1367 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x84f + +goroutine 1407 [select]: +runtime.gopark(0xc002b9cfb0?, 0x2?, 0xb4?, 0x1b?, 0xc002b9cf74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002b9cde8 sp=0xc002b9cdc8 pc=0x4777ae +runtime.selectgo(0xc002b9cfb0, 0xc002b9cf70, 0xa3939202d202020?, 0x0, 0x31202d202020200a?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002b9cf38 sp=0xc002b9cde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002b9cfe0 sp=0xc002b9cf38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002b9cfe8 sp=0xc002b9cfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1367 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1408 [select]: +runtime.gopark(0xc002b9dfb0?, 0x2?, 0xb0?, 0xdd?, 0xc002b9dee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002b9dd58 sp=0xc002b9dd38 pc=0x4777ae +runtime.selectgo(0xc002b9dfb0, 0xc002b9dee0, 0x20200a3038202d20?, 0x0, 0xa3038202d202020?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002b9dea8 sp=0xc002b9dd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002b9dfe0 sp=0xc002b9dea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002b9dfe8 sp=0xc002b9dfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1367 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1409 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002b9ed60 sp=0xc002b9ed40 pc=0x4777ae +runtime.chanrecv(0xc002bf63c0, 0xc002b9ee60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002b9edd8 sp=0xc002b9ed60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002b9ee00 sp=0xc002b9edd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002b9efe0 sp=0xc002b9ee00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002b9efe8 sp=0xc002b9efe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1367 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1458 [select]: +runtime.gopark(0xc0024dea68?, 0x2?, 0xfe?, 0x1c?, 0xc0024dea14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024de880 sp=0xc0024de860 pc=0x4777ae +runtime.selectgo(0xc0024dea68, 0xc0024dea10, 0x7f429a48c700?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0024de9d0 sp=0xc0024de880 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0026a5900, {0xc0026ab870, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0024dea98 sp=0xc0024de9d0 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0026a5900, {0xc0026ab870, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0024deb30 sp=0xc0024dea98 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0026ab4a0, {0xc0026ab870, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0024deb78 sp=0xc0024deb30 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0026ab4a0}, {0xc0026ab870, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0024debc0 sp=0xc0024deb78 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d57560, {0xc0026ab870, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0024dec30 sp=0xc0024debc0 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0026ab860, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0024ded00 sp=0xc0024dec30 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x98b2c5?, 0xc001d57560, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0024dedf0 sp=0xc0024ded00 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0026ab860, {0x7f424347ae80, 0x779c980}, 0x6aa1293eacb061af?, {0x0, 0x0}, {0x43dd860, 0xc0026a5bd0}, 0x2a1b853c16d73cc7?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0024deea8 sp=0xc0024dedf0 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc0016704b0, {0x43dd860?, 0xc0026a5bd0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0024df0e0 sp=0xc0024deea8 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0026ab9b0, {0x43dd860, 0xc0026a5bd0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0024df1c8 sp=0xc0024df0e0 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002bf80e0, {0x43dd860, 0xc0026a5bd0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0024df200 sp=0xc0024df1c8 pc=0x1fd94b2 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamServer).Recv(0xc002688760) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:167 +0x66 fp=0xc0024df240 sp=0xc0024df200 pc=0x2102886 +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream(0xc0025cdda0, {0x5b512b8, 0xc002688760}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:87 +0x218 fp=0xc0024df370 sp=0xc0024df240 pc=0x2154718 +github.com/hashicorp/go-plugin/internal/plugin._GRPCBroker_StartStream_Handler({0x41cb860?, 0xc0025cdda0}, {0x5b4a6f0?, 0xc002bf80e0}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:148 +0xb6 fp=0xc0024df3b0 sp=0xc0024df370 pc=0x2102716 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41cb860, 0xc0025cdda0}, {0x5b4ab30, 0xc0026ab9b0}, 0xc000dc2558?, 0x47efc98) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0024df400 sp=0xc0024df3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41cb860, 0xc0025cdda0}, {0x5b4ab30, 0xc0026ab9b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0024df490 sp=0xc0024df400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41cb860, 0xc0025cdda0}, {0x5b492f8, 0xc0016704b0}, 0xc000dc2558, 0xc002452480) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0024df7d0 sp=0xc0024df490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41cb860, 0xc0025cdda0}, {0x5b492f8, 0xc0016704b0}, 0x3f252a0?, 0xc002688690?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0024df848 sp=0xc0024df7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d57560, 0xc0025cde30, 0x770cb80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0024dfdb0 sp=0xc0024df848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d57560, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0024dff28 sp=0xc0024dfdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0024dffe0 sp=0xc0024dff28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024dffe8 sp=0xc0024dffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1398 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1459 [select]: +runtime.gopark(0xc002c17310?, 0x3?, 0xd9?, 0xea?, 0xc002c17292?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c170f0 sp=0xc002c170d0 pc=0x4777ae +runtime.selectgo(0xc002c17310, 0xc002c1728c, 0x46216a2?, 0x0, 0x4?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c17240 sp=0xc002c170f0 pc=0x48916b +github.com/hashicorp/go-plugin.(*grpcStdioServer).StreamStdio(0xc0026e3900, 0x42219c0?, {0x5b4ebb0, 0xc002688840}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:61 +0x1e5 fp=0xc002c17350 sp=0xc002c17240 pc=0x215bbc5 +github.com/hashicorp/go-plugin/internal/plugin._GRPCStdio_StreamStdio_Handler({0x409e580?, 0xc0026e3900}, {0x5b4a6f0, 0xc002bf8140}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:185 +0xf7 fp=0xc002c173b0 sp=0xc002c17350 pc=0x2104077 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x409e580, 0xc0026e3900}, {0x5b4ab30, 0xc0026abbf0}, 0xc000dc25a0?, 0x47efca8) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002c17400 sp=0xc002c173b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x409e580, 0xc0026e3900}, {0x5b4ab30, 0xc0026abbf0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002c17490 sp=0xc002c17400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x409e580, 0xc0026e3900}, {0x5b492f8, 0xc0016705a0}, 0xc000dc25a0, 0xc002452600) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002c177d0 sp=0xc002c17490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x409e580, 0xc0026e3900}, {0x5b492f8, 0xc0016705a0}, 0x3f252a0?, 0xc002688770?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002c17848 sp=0xc002c177d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d57680, 0xc002780000, 0x770cba0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002c17db0 sp=0xc002c17848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d57680, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002c17f28 sp=0xc002c17db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002c17fe0 sp=0xc002c17f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c17fe8 sp=0xc002c17fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1398 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1460 [select]: +runtime.gopark(0xc00218c890?, 0x2?, 0x98?, 0xc7?, 0xc00218c83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00218c6a8 sp=0xc00218c688 pc=0x4777ae +runtime.selectgo(0xc00218c890, 0xc00218c838, 0x7f429a48c100?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00218c7f8 sp=0xc00218c6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0026a5ae0, {0xc0026abcf0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc00218c8c0 sp=0xc00218c7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0026a5ae0, {0xc0026abcf0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc00218c958 sp=0xc00218c8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0026ab7a0, {0xc0026abcf0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc00218c9a0 sp=0xc00218c958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0026ab7a0}, {0xc0026abcf0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00218c9e8 sp=0xc00218c9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d577a0, {0xc0026abcf0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc00218ca58 sp=0xc00218c9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0026abce0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc00218cb28 sp=0xc00218ca58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc002452e00?, 0xc001d577a0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc00218cc18 sp=0xc00218cb28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0026abce0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc002bfc050}, 0xc0025cdad0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc00218ccd0 sp=0xc00218cc18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001670690, {0x43dcf60?, 0xc002bfc050}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc00218cf08 sp=0xc00218ccd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0026abe30, {0x43dcf60, 0xc002bfc050}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc00218cff0 sp=0xc00218cf08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002bf81a0, {0x43dcf60, 0xc002bfc050}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc00218d028 sp=0xc00218cff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc002688920) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc00218d068 sp=0xc00218d028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc002688920?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc00218d0a8 sp=0xc00218d068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc002688930) + :1 +0x46 fp=0xc00218d0e0 sp=0xc00218d0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0025cdc50, {0x5b51368, 0xc002688930}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc00218d320 sp=0xc00218d0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc0026e2fa0, {0x5b51038, 0xc002688920}) + :1 +0xaf fp=0xc00218d370 sp=0xc00218d320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc0026e2fa0}, {0x5b4a6f0?, 0xc002bf81a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc00218d3b0 sp=0xc00218d370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc0026e2fa0}, {0x5b4ab30, 0xc0026abe30}, 0xc000dc25e8?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc00218d400 sp=0xc00218d3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc0026e2fa0}, {0x5b4ab30, 0xc0026abe30}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc00218d490 sp=0xc00218d400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc0026e2fa0}, {0x5b492f8, 0xc001670690}, 0xc000dc25e8, 0xc002452780) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc00218d7d0 sp=0xc00218d490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc0026e2fa0}, {0x5b492f8, 0xc001670690}, 0x3f252a0?, 0xc002688850?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc00218d848 sp=0xc00218d7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d577a0, 0xc0025cdce0, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc00218ddb0 sp=0xc00218d848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d577a0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc00218df28 sp=0xc00218ddb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc00218dfe0 sp=0xc00218df28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00218dfe8 sp=0xc00218dfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1398 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1461 [select]: +runtime.gopark(0xc002c0bfa0?, 0x3?, 0x0?, 0x4e?, 0xc002c0bf3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c0bda8 sp=0xc002c0bd88 pc=0x4777ae +runtime.selectgo(0xc002c0bfa0, 0xc002c0bf34, 0xc002c0bf40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c0bef8 sp=0xc002c0bda8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:73 +0x1ad fp=0xc002c0bfe0 sp=0xc002c0bef8 pc=0x2154aad +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c0bfe8 sp=0xc002c0bfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream in goroutine 1458 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:71 +0x1d1 + +goroutine 1467 [select]: +runtime.gopark(0xc001bce890?, 0x2?, 0x98?, 0xe7?, 0xc001bce83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bce6a8 sp=0xc001bce688 pc=0x4777ae +runtime.selectgo(0xc001bce890, 0xc001bce838, 0x7f429a48bb00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001bce7f8 sp=0xc001bce6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002bfc5a0, {0xc002bfe580, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc001bce8c0 sp=0xc001bce7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002bfc5a0, {0xc002bfe580, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001bce958 sp=0xc001bce8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002bfe4e0, {0xc002bfe580, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001bce9a0 sp=0xc001bce958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002bfe4e0}, {0xc002bfe580, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001bce9e8 sp=0xc001bce9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d57d40, {0xc002bfe580, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001bcea58 sp=0xc001bce9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002bfe570, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001bceb28 sp=0xc001bcea58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc002453b80?, 0xc001d57d40, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001bcec18 sp=0xc001bceb28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002bfe570, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc002bfc960}, 0xc0025cdad0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001bcecd0 sp=0xc001bcec18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001670780, {0x43dcf60?, 0xc002bfc960}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001bcef08 sp=0xc001bcecd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002bfe6c0, {0x43dcf60, 0xc002bfc960}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001bceff0 sp=0xc001bcef08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002bf8380, {0x43dcf60, 0xc002bfc960}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001bcf028 sp=0xc001bceff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc002688d10) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001bcf068 sp=0xc001bcf028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc002688d10?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc001bcf0a8 sp=0xc001bcf068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc002688d20) + :1 +0x46 fp=0xc001bcf0e0 sp=0xc001bcf0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0025cdc50, {0x5b51368, 0xc002688d20}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001bcf320 sp=0xc001bcf0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc0026e2fa0, {0x5b51038, 0xc002688d10}) + :1 +0xaf fp=0xc001bcf370 sp=0xc001bcf320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc0026e2fa0}, {0x5b4a6f0?, 0xc002bf8380}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc001bcf3b0 sp=0xc001bcf370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc0026e2fa0}, {0x5b4ab30, 0xc002bfe6c0}, 0xc000dc2c00?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001bcf400 sp=0xc001bcf3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc0026e2fa0}, {0x5b4ab30, 0xc002bfe6c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001bcf490 sp=0xc001bcf400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc0026e2fa0}, {0x5b492f8, 0xc001670780}, 0xc000dc2c00, 0xc002453500) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001bcf7d0 sp=0xc001bcf490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc0026e2fa0}, {0x5b492f8, 0xc001670780}, 0x3f252a0?, 0xc002688c40?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001bcf848 sp=0xc001bcf7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d57d40, 0xc0025cdce0, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001bcfdb0 sp=0xc001bcf848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d57d40, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001bcff28 sp=0xc001bcfdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001bcffe0 sp=0xc001bcff28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bcffe8 sp=0xc001bcffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1398 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1463 [select]: +runtime.gopark(0xc002c0dfb0?, 0x2?, 0xb4?, 0x1b?, 0xc002c0df74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c0dde8 sp=0xc002c0ddc8 pc=0x4777ae +runtime.selectgo(0xc002c0dfb0, 0xc002c0df70, 0x0?, 0x0, 0xc002000000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c0df38 sp=0xc002c0dde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002c0dfe0 sp=0xc002c0df38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c0dfe8 sp=0xc002c0dfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1367 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1464 [select]: +runtime.gopark(0xc002c0efb0?, 0x2?, 0xb0?, 0xed?, 0xc002c0eee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c0ed58 sp=0xc002c0ed38 pc=0x4777ae +runtime.selectgo(0xc002c0efb0, 0xc002c0eee0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c0eea8 sp=0xc002c0ed58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002c0efe0 sp=0xc002c0eea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c0efe8 sp=0xc002c0efe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1367 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1465 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c21d60 sp=0xc002c21d40 pc=0x4777ae +runtime.chanrecv(0xc002bf68a0, 0xc002c21e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c21dd8 sp=0xc002c21d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c21e00 sp=0xc002c21dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002c21fe0 sp=0xc002c21e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c21fe8 sp=0xc002c21fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1367 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1473 [select]: +runtime.gopark(0xc002bae890?, 0x2?, 0x98?, 0xe7?, 0xc002bae83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002bae6a8 sp=0xc002bae688 pc=0x4777ae +runtime.selectgo(0xc002bae890, 0xc002bae838, 0x7f429a48af00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002bae7f8 sp=0xc002bae6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002bfceb0, {0xc002bfedc0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002bae8c0 sp=0xc002bae7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002bfceb0, {0xc002bfedc0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002bae958 sp=0xc002bae8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002bfecf0, {0xc002bfedc0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002bae9a0 sp=0xc002bae958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002bfecf0}, {0xc002bfedc0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002bae9e8 sp=0xc002bae9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d8a360, {0xc002bfedc0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002baea58 sp=0xc002bae9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002bfedb0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002baeb28 sp=0xc002baea58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc00236c8c0?, 0xc001d8a360, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002baec18 sp=0xc002baeb28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002bfedb0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc002bfd270}, 0xc0025cdad0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002baecd0 sp=0xc002baec18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001670870, {0x43dcf60?, 0xc002bfd270}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002baef08 sp=0xc002baecd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002bfef00, {0x43dcf60, 0xc002bfd270}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002baeff0 sp=0xc002baef08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002bf8580, {0x43dcf60, 0xc002bfd270}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002baf028 sp=0xc002baeff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc0026890e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc002baf068 sp=0xc002baf028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc0026890e0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc002baf0a8 sp=0xc002baf068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc0026890f0) + :1 +0x46 fp=0xc002baf0e0 sp=0xc002baf0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0025cdc50, {0x5b51368, 0xc0026890f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc002baf320 sp=0xc002baf0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc0026e2fa0, {0x5b51038, 0xc0026890e0}) + :1 +0xaf fp=0xc002baf370 sp=0xc002baf320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc0026e2fa0}, {0x5b4a6f0?, 0xc002bf8580}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc002baf3b0 sp=0xc002baf370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc0026e2fa0}, {0x5b4ab30, 0xc002bfef00}, 0xc000dc3128?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002baf400 sp=0xc002baf3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc0026e2fa0}, {0x5b4ab30, 0xc002bfef00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002baf490 sp=0xc002baf400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc0026e2fa0}, {0x5b492f8, 0xc001670870}, 0xc000dc3128, 0xc00236c240) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002baf7d0 sp=0xc002baf490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc0026e2fa0}, {0x5b492f8, 0xc001670870}, 0x3f252a0?, 0xc002689010?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002baf848 sp=0xc002baf7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d8a360, 0xc0025cdce0, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002bafdb0 sp=0xc002baf848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d8a360, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002baff28 sp=0xc002bafdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002baffe0 sp=0xc002baff28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002baffe8 sp=0xc002baffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1398 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1469 [select]: +runtime.gopark(0xc002c0afb0?, 0x2?, 0xb4?, 0x1b?, 0xc002c0af74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c0ade8 sp=0xc002c0adc8 pc=0x4777ae +runtime.selectgo(0xc002c0afb0, 0xc002c0af70, 0xc002bf60c0?, 0x0, 0xc002780810?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c0af38 sp=0xc002c0ade8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002c0afe0 sp=0xc002c0af38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c0afe8 sp=0xc002c0afe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1367 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1470 [select]: +runtime.gopark(0xc002c1dfb0?, 0x2?, 0xb0?, 0xdd?, 0xc002c1dee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c1dd58 sp=0xc002c1dd38 pc=0x4777ae +runtime.selectgo(0xc002c1dfb0, 0xc002c1dee0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c1dea8 sp=0xc002c1dd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002c1dfe0 sp=0xc002c1dea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c1dfe8 sp=0xc002c1dfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1367 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1471 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c08d60 sp=0xc002c08d40 pc=0x4777ae +runtime.chanrecv(0xc002bf6cc0, 0xc002c08e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c08dd8 sp=0xc002c08d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c08e00 sp=0xc002c08dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002c08fe0 sp=0xc002c08e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c08fe8 sp=0xc002c08fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1367 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1478 [chan receive]: +runtime.gopark(0x7769700?, 0x460a1ee?, 0x5?, 0x0?, 0x462ff63?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020f3b90 sp=0xc0020f3b70 pc=0x4777ae +runtime.chanrecv(0xc002666b40, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0020f3c08 sp=0xc0020f3b90 pc=0x441325 +runtime.chanrecv1(0x4b05a9?, 0xc0020f3c58?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0020f3c30 sp=0xc0020f3c08 pc=0x440f72 +github.com/rancher/opni/pkg/util/future.(*future[...]).Get(...) + /home/ec2-user/opni/pkg/util/future/future.go:61 +github.com/rancher/opni/pkg/util/future.(*future[...]).Get() + /home/ec2-user/opni/pkg/util/future/future.go:60 +0x5a fp=0xc0020f3c68 sp=0xc0020f3c30 pc=0x201331a +github.com/rancher/opni/pkg/agent/v2.(*Agent).runGatewayClient(0xc0025ee340, {0x5b426e0, 0xc002c2e550}) + /home/ec2-user/opni/pkg/agent/v2/agent.go:522 +0x34c fp=0xc0020f3f50 sp=0xc0020f3c68 pc=0x21c052c +github.com/rancher/opni/pkg/agent/v2.(*Agent).ListenAndServe.func3() + /home/ec2-user/opni/pkg/agent/v2/agent.go:446 +0x47 fp=0xc0020f3fa0 sp=0xc0020f3f50 pc=0x21bf167 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0020f3fe0 sp=0xc0020f3fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020f3fe8 sp=0xc0020f3fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1320 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1479 [select]: +runtime.gopark(0xc002c09fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002c09f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c09de8 sp=0xc002c09dc8 pc=0x4777ae +runtime.selectgo(0xc002c09fb0, 0xc002c09f70, 0x21bf167?, 0x0, 0x5b426e0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c09f38 sp=0xc002c09de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002c09fe0 sp=0xc002c09f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c09fe8 sp=0xc002c09fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1478 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1422 [select]: +runtime.gopark(0xc0029b4fb0?, 0x2?, 0xb0?, 0x4d?, 0xc0029b4ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029b4d58 sp=0xc0029b4d38 pc=0x4777ae +runtime.selectgo(0xc0029b4fb0, 0xc0029b4ee0, 0xc000a8ef00?, 0x0, 0x1174725?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029b4ea8 sp=0xc0029b4d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0029b4fe0 sp=0xc0029b4ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029b4fe8 sp=0xc0029b4fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1478 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1423 [chan receive]: +runtime.gopark(0xc00160e7c0?, 0xc0016190e0?, 0x70?, 0x7?, 0xc001340780?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029b2d60 sp=0xc0029b2d40 pc=0x4777ae +runtime.chanrecv(0xc0022fbec0, 0xc0029b2e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0029b2dd8 sp=0xc0029b2d60 pc=0x441325 +runtime.chanrecv1(0xc000770a10?, 0xc00086d110?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0029b2e00 sp=0xc0029b2dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc0029b2fe0 sp=0xc0029b2e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029b2fe8 sp=0xc0029b2fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1478 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1424 [select]: +runtime.gopark(0xc002c1afb0?, 0x2?, 0xa9?, 0x5?, 0xc002c1af2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c1ada8 sp=0xc002c1ad88 pc=0x4777ae +runtime.selectgo(0xc002c1afb0, 0xc002c1af28, 0x4b0571?, 0x0, 0x11847d3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c1aef8 sp=0xc002c1ada8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002c1afe0 sp=0xc002c1aef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c1afe8 sp=0xc002c1afe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1425 [select]: +runtime.gopark(0xc002c8bf90?, 0x2?, 0xff?, 0xff?, 0xc002c8beac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c8bd28 sp=0xc002c8bd08 pc=0x4777ae +runtime.selectgo(0xc002c8bf90, 0xc002c8bea8, 0x4b0571?, 0x0, 0xc000f43718?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c8be78 sp=0xc002c8bd28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc002949680) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002c8bfc0 sp=0xc002c8be78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002c8bfe0 sp=0xc002c8bfc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c8bfe8 sp=0xc002c8bfe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1490 [select]: +runtime.gopark(0xc002ec7f40?, 0x2?, 0xb4?, 0x1b?, 0xc002ec7ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ec7d48 sp=0xc002ec7d28 pc=0x4777ae +runtime.selectgo(0xc002ec7f40, 0xc002ec7ec8, 0xc002711470?, 0x0, 0x5655434a55573374?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ec7e98 sp=0xc002ec7d48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002846d20}, 0xc002949710, 0x0, 0xc00245ee40, 0xc002577b50) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc002ec7f70 sp=0xc002ec7e98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc002ec7fe0 sp=0xc002ec7f70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ec7fe8 sp=0xc002ec7fe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1491 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0xbe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c1be20 sp=0xc002c1be00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c1be98 sp=0xc002c1be20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x2166b62?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c1bec0 sp=0xc002c1be98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c1bfe0 sp=0xc002c1bec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c1bfe8 sp=0xc002c1bfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1492 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x7e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c87e20 sp=0xc002c87e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c87e98 sp=0xc002c87e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x5b058a8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c87ec0 sp=0xc002c87e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c87fe0 sp=0xc002c87ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c87fe8 sp=0xc002c87fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1493 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x8e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c88e20 sp=0xc002c88e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c88e98 sp=0xc002c88e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c88ec0 sp=0xc002c88e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c88fe0 sp=0xc002c88ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c88fe8 sp=0xc002c88fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1494 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x9e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c89e20 sp=0xc002c89e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c89e98 sp=0xc002c89e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x4166b20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c89ec0 sp=0xc002c89e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c89fe0 sp=0xc002c89ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c89fe8 sp=0xc002c89fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1495 [select]: +runtime.gopark(0xc002c8afb0?, 0x2?, 0xb4?, 0x1b?, 0xc002c8af74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c8ade8 sp=0xc002c8adc8 pc=0x4777ae +runtime.selectgo(0xc002c8afb0, 0xc002c8af70, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c8af38 sp=0xc002c8ade8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002c8afe0 sp=0xc002c8af38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c8afe8 sp=0xc002c8afe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1478 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1496 [select]: +runtime.gopark(0xc002d0bfb0?, 0x2?, 0x0?, 0x0?, 0xc002d0bee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d0bd58 sp=0xc002d0bd38 pc=0x4777ae +runtime.selectgo(0xc002d0bfb0, 0xc002d0bee0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d0bea8 sp=0xc002d0bd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002d0bfe0 sp=0xc002d0bea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d0bfe8 sp=0xc002d0bfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1478 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1497 [chan receive]: +runtime.gopark(0xc0009f2430?, 0xc000992790?, 0xd0?, 0x27?, 0xc000992810?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029b6d60 sp=0xc0029b6d40 pc=0x4777ae +runtime.chanrecv(0xc002858300, 0xc0029b6e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0029b6dd8 sp=0xc0029b6d60 pc=0x441325 +runtime.chanrecv1(0xc000807a00?, 0xc0009d3420?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0029b6e00 sp=0xc0029b6dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc0029b6fe0 sp=0xc0029b6e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029b6fe8 sp=0xc0029b6fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1478 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1498 [select]: +runtime.gopark(0xc002d07fb0?, 0x2?, 0x0?, 0x7e?, 0xc002d07f2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d07da8 sp=0xc002d07d88 pc=0x4777ae +runtime.selectgo(0xc002d07fb0, 0xc002d07f28, 0x4b0571?, 0x0, 0x11847d3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d07ef8 sp=0xc002d07da8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002d07fe0 sp=0xc002d07ef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d07fe8 sp=0xc002d07fe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1499 [select]: +runtime.gopark(0xc002d09f90?, 0x2?, 0xb4?, 0x1b?, 0xc002d09eac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d09d28 sp=0xc002d09d08 pc=0x4777ae +runtime.selectgo(0xc002d09f90, 0xc002d09ea8, 0x4b0571?, 0x0, 0x4b0571?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d09e78 sp=0xc002d09d28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc002949800) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002d09fc0 sp=0xc002d09e78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002d09fe0 sp=0xc002d09fc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d09fe8 sp=0xc002d09fe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1500 [select]: +runtime.gopark(0xc002babf40?, 0x2?, 0xb4?, 0x1b?, 0xc002babecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002babd48 sp=0xc002babd28 pc=0x4777ae +runtime.selectgo(0xc002babf40, 0xc002babec8, 0x4b0571?, 0x0, 0x8ab66c?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002babe98 sp=0xc002babd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002847680}, 0xc002949890, 0x0, 0xc0022ee1e0, 0xc0029088c0) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc002babf70 sp=0xc002babe98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc002babfe0 sp=0xc002babf70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002babfe8 sp=0xc002babfe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1501 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc0028581e0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d05e20 sp=0xc002d05e00 pc=0x4777ae +runtime.chanrecv(0xc0028581e0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d05e98 sp=0xc002d05e20 pc=0x441325 +runtime.chanrecv1(0xc002849cb0?, 0x8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d05ec0 sp=0xc002d05e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d05fe0 sp=0xc002d05ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d05fe8 sp=0xc002d05fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1502 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc0028581e0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d0ae20 sp=0xc002d0ae00 pc=0x4777ae +runtime.chanrecv(0xc0028581e0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d0ae98 sp=0xc002d0ae20 pc=0x441325 +runtime.chanrecv1(0xc002849cb0?, 0x6a55575255575268?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d0aec0 sp=0xc002d0ae98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d0afe0 sp=0xc002d0aec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d0afe8 sp=0xc002d0afe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1503 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x3e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cb3e20 sp=0xc002cb3e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cb3e98 sp=0xc002cb3e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x2166b62?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cb3ec0 sp=0xc002cb3e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cb3fe0 sp=0xc002cb3ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cb3fe8 sp=0xc002cb3fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1504 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x4e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d04e20 sp=0xc002d04e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d04e98 sp=0xc002d04e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x2005f24?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d04ec0 sp=0xc002d04e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d04fe0 sp=0xc002d04ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d04fe8 sp=0xc002d04fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1505 [select]: +runtime.gopark(0xc002b95c98?, 0x2?, 0x67?, 0x6a?, 0xc002b95bb4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002b95a28 sp=0xc002b95a08 pc=0x4777ae +runtime.selectgo(0xc002b95c98, 0xc002b95bb0, 0xc002574ea0?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002b95b78 sp=0xc002b95a28 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc0024e9220) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc002b95cc8 sp=0xc002b95b78 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc001c96b40, {0x5b426a8, 0xc002849cb0}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc002b95ee8 sp=0xc002b95cc8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Splice.func2() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:190 +0x6b fp=0xc002b95fe0 sp=0xc002b95ee8 pc=0x136faab +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002b95fe8 sp=0xc002b95fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Splice in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:189 +0x61a + +goroutine 1486 [semacquire]: +runtime.gopark(0x473391?, 0xc002771238?, 0xa0?, 0x57?, 0x448485?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0027711c8 sp=0xc0027711a8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +runtime.semacquire1(0xc002f90290, 0x58?, 0x1, 0x0, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:160 +0x218 fp=0xc002771230 sp=0xc0027711c8 pc=0x48a478 +sync.runtime_Semacquire(0xc002f90290?) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:62 +0x25 fp=0xc002771268 sp=0xc002771230 pc=0x4a8fc5 +sync.(*WaitGroup).Wait(0xc002f90288) + /home/ec2-user/sdk/go1.21.0/src/sync/waitgroup.go:116 +0xa5 fp=0xc002771298 sp=0xc002771268 pc=0x4c06e5 +golang.org/x/sync/errgroup.(*Group).Wait(0xc002f90280) + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:53 +0x38 fp=0xc0027712c8 sp=0xc002771298 pc=0x20071d8 +github.com/rancher/opni/pkg/management.NewServer.unknownServiceHandler.func3({0xc000ef57c0?, 0x5b426a8?}, {0x5b4ab30?, 0xc002c31a40}) + /home/ec2-user/opni/pkg/management/extensions.go:567 +0xa85 fp=0xc002771508 sp=0xc0027712c8 pc=0x231d105 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x0, 0x0}, {0x5b492f8, 0xc001670e10}, 0xc001e465d0, 0xc00158a890) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002771848 sp=0xc002771508 pc=0x12b0e02 +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c1e0, {0x5b54e00, 0xc002c26d00}, 0xc001d8afc0, 0x0, 0xc001ded2a0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002771db0 sp=0xc002771848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c1e0, {0x5b54e00, 0xc002c26d00}, 0xc001d8afc0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1747 +0xf15 fp=0xc002771f28 sp=0xc002771db0 pc=0x11846f5 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002771fe0 sp=0xc002771f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002771fe8 sp=0xc002771fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1484 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1457 [select]: +runtime.gopark(0xc002bb3f40?, 0x2?, 0xb4?, 0x1b?, 0xc002bb3ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002bb3d48 sp=0xc002bb3d28 pc=0x4777ae +runtime.selectgo(0xc002bb3f40, 0xc002bb3ec8, 0xc002849bc0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002bb3e98 sp=0xc002bb3d48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002669090}, 0xc0015ec2d0, 0x0, 0xc00251db00, 0xc002d00550) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc002bb3f70 sp=0xc002bb3e98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc002bb3fe0 sp=0xc002bb3f70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002bb3fe8 sp=0xc002bb3fe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1454 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1506 [chan receive]: +runtime.gopark(0xc00266d770?, 0x4b05a9?, 0x50?, 0xbe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00234be20 sp=0xc00234be00 pc=0x4777ae +runtime.chanrecv(0xc002666480, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00234be98 sp=0xc00234be20 pc=0x441325 +runtime.chanrecv1(0xc00266da40?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00234bec0 sp=0xc00234be98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc00234bfe0 sp=0xc00234bec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00234bfe8 sp=0xc00234bfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1454 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1507 [chan receive]: +runtime.gopark(0xc00266d770?, 0x4b05a9?, 0x50?, 0x6e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017d6e20 sp=0xc0017d6e00 pc=0x4777ae +runtime.chanrecv(0xc002666480, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0017d6e98 sp=0xc0017d6e20 pc=0x441325 +runtime.chanrecv1(0xc00266da40?, 0x4166b20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0017d6ec0 sp=0xc0017d6e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc0017d6fe0 sp=0xc0017d6ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017d6fe8 sp=0xc0017d6fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1454 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1508 [chan receive]: +runtime.gopark(0xc00266d770?, 0x4b05a9?, 0x50?, 0x5e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c85e20 sp=0xc002c85e00 pc=0x4777ae +runtime.chanrecv(0xc002666480, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c85e98 sp=0xc002c85e20 pc=0x441325 +runtime.chanrecv1(0xc00266da40?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c85ec0 sp=0xc002c85e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c85fe0 sp=0xc002c85ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c85fe8 sp=0xc002c85fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1454 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1509 [chan receive]: +runtime.gopark(0xc00266d770?, 0x4b05a9?, 0x50?, 0x6e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c86e20 sp=0xc002c86e00 pc=0x4777ae +runtime.chanrecv(0xc002666480, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c86e98 sp=0xc002c86e20 pc=0x441325 +runtime.chanrecv1(0xc00266da40?, 0x5b058a8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c86ec0 sp=0xc002c86e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c86fe0 sp=0xc002c86ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c86fe8 sp=0xc002c86fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1454 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1523 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc0028581e0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d08e20 sp=0xc002d08e00 pc=0x4777ae +runtime.chanrecv(0xc0028581e0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d08e98 sp=0xc002d08e20 pc=0x441325 +runtime.chanrecv1(0xc002849cb0?, 0x1000000010000?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d08ec0 sp=0xc002d08e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d08fe0 sp=0xc002d08ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d08fe8 sp=0xc002d08fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1482 [select]: +runtime.gopark(0xc002c5de48?, 0x2?, 0x0?, 0x0?, 0xc002c5ddcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c5dc30 sp=0xc002c5dc10 pc=0x4777ae +runtime.selectgo(0xc002c5de48, 0xc002c5ddc8, 0xc002c5ddd8?, 0x0, 0xc001d6f418?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c5dd80 sp=0xc002c5dc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002c2ec30, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002c5de78 sp=0xc002c5dd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0026af880) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002c5df30 sp=0xc002c5de78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc002c5dfe0 sp=0xc002c5df30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c5dfe8 sp=0xc002c5dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1481 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1483 [select]: +runtime.gopark(0xc002c1ef70?, 0x4?, 0xf8?, 0xeb?, 0xc002c1edd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c1ec00 sp=0xc002c1ebe0 pc=0x4777ae +runtime.selectgo(0xc002c1ef70, 0xc002c1edd0, 0x40a16a0?, 0x0, 0xc0026f2dd0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c1ed50 sp=0xc002c1ec00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002c26d00) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc002c1efc0 sp=0xc002c1ed50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc002c1efe0 sp=0xc002c1efc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c1efe8 sp=0xc002c1efe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1481 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1484 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c3d7d8 sp=0xc002c3d7b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002c3d810 sp=0xc002c3d7d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c44c8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002c3d830 sp=0xc002c3d810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc002c19620, 0xc002c3e000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002c3d878 sp=0xc002c3d830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc002c19600, {0xc002c3e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc002c3d968 sp=0xc002c3d878 pc=0x50e5a5 +net.(*netFD).Read(0xc002c19600, {0xc002c3e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc002c3d9c8 sp=0xc002c3d968 pc=0x89342b +net.(*conn).Read(0xc001d6f418, {0xc002c3e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc002c3da50 sp=0xc002c3d9c8 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6f418, {0xc002c3e000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc002c3da98 sp=0xc002c3da50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc002574fc0, {0xc002686820, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc002c3db58 sp=0xc002c3da98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc002574fc0}, {0xc002686820, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002c3dba0 sp=0xc002c3db58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc002686820, 0x9, 0x9}, {0x5b0f0a0, 0xc002574fc0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc002c3dc40 sp=0xc002c3dba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0026867e0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc002c3dd78 sp=0xc002c3dc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002c26d00, 0xc000b3d0e0?, 0x28?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc002c3dec8 sp=0xc002c3dd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c1e0, {0x5b54e00?, 0xc002c26d00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc002c3df80 sp=0xc002c3dec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc002c3dfe0 sp=0xc002c3df80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c3dfe8 sp=0xc002c3dfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1481 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1522 [select]: +runtime.gopark(0xc002c153f8?, 0x2?, 0xc0?, 0x72?, 0xc002c1537c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c151e8 sp=0xc002c151c8 pc=0x4777ae +runtime.selectgo(0xc002c153f8, 0xc002c15378, 0x5b203e8?, 0x0, 0x2710?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c15338 sp=0xc002c151e8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc002847590, {0xc002849db0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc002c15428 sp=0xc002c15338 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002847590, {0xc002849db0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc002c154c0 sp=0xc002c15428 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002849d70, {0xc002849db0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002c15508 sp=0xc002c154c0 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002849d70}, {0xc002849db0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002c15550 sp=0xc002c15508 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c96a20, {0xc002849db0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002c155c0 sp=0xc002c15550 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002849da0, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002c15690 sp=0xc002c155c0 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc001c96a20, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002c15780 sp=0xc002c15690 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002849da0, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43f1c20, 0xc002fe0310}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002c15838 sp=0xc002c15780 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc002713d40, {0x43f1c20?, 0xc002fe0310}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc002c15a98 sp=0xc002c15838 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001c96534?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc002c15af8 sp=0xc002c15a98 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001c96480, 0xc002c15cb0, 0xc002c15c80) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc002c15ba0 sp=0xc002c15af8 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001c96480, {0x43f1c20, 0xc002fe0310}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc002c15ce0 sp=0xc002c15ba0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc0021e7f80, {0x43f1c20, 0xc002fe0310}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc002c15ea8 sp=0xc002c15ce0 pc=0x12ac5b9 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectClient).Recv(0xc002908810) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:71 +0x66 fp=0xc002c15ee8 sp=0xc002c15ea8 pc=0x2003b86 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc002c15fe0 sp=0xc002c15ee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c15fe8 sp=0xc002c15fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1505 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1511 [select]: +runtime.gopark(0xc0029bfaa8?, 0x2?, 0x67?, 0x6a?, 0xc0029bf9c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029bf838 sp=0xc0029bf818 pc=0x4777ae +runtime.selectgo(0xc0029bfaa8, 0xc0029bf9c0, 0xc0029bf9d0?, 0x0, 0x43f1c20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029bf988 sp=0xc0029bf838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc002650f00) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc0029bfad8 sp=0xc0029bf988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc001c58480, {0x5b426a8, 0xc00266da40}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc0029bfcf8 sp=0xc0029bfad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc0029bffe0 sp=0xc0029bfcf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029bffe8 sp=0xc0029bffe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 1510 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 1436 [select]: +runtime.gopark(0xc002cacf90?, 0x2?, 0xb4?, 0x1b?, 0xc002caceac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cacd28 sp=0xc002cacd08 pc=0x4777ae +runtime.selectgo(0xc002cacf90, 0xc002cacea8, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cace78 sp=0xc002cacd28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc0004f1b00) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002cacfc0 sp=0xc002cace78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002cacfe0 sp=0xc002cacfc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cacfe8 sp=0xc002cacfe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1437 [select]: +runtime.gopark(0xc002ecbf40?, 0x2?, 0xb4?, 0x1b?, 0xc002ecbecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ecbd48 sp=0xc002ecbd28 pc=0x4777ae +runtime.selectgo(0xc002ecbf40, 0xc002ecbec8, 0x4ad421?, 0x0, 0x1381849?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ecbe98 sp=0xc002ecbd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc0027ca5f0}, 0xc0004f1b90, 0x0, 0xc002047680, 0xc002857840) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc002ecbf70 sp=0xc002ecbe98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc002ecbfe0 sp=0xc002ecbf70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ecbfe8 sp=0xc002ecbfe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1438 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x3e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d83e20 sp=0xc002d83e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d83e98 sp=0xc002d83e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x4166b20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d83ec0 sp=0xc002d83e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d83fe0 sp=0xc002d83ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d83fe8 sp=0xc002d83fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1439 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x5e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d85e20 sp=0xc002d85e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d85e98 sp=0xc002d85e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x5b058a8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d85ec0 sp=0xc002d85e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d85fe0 sp=0xc002d85ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d85fe8 sp=0xc002d85fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1440 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x4e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d84e20 sp=0xc002d84e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d84e98 sp=0xc002d84e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x2154feb?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d84ec0 sp=0xc002d84e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d84fe0 sp=0xc002d84ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d84fe8 sp=0xc002d84fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1441 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0xde?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cade20 sp=0xc002cade00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cade98 sp=0xc002cade20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cadec0 sp=0xc002cade98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cadfe0 sp=0xc002cadec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cadfe8 sp=0xc002cadfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1512 [select]: +runtime.gopark(0xc0017c7710?, 0x2?, 0x0?, 0x0?, 0xc0017c76bc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017c7528 sp=0xc0017c7508 pc=0x4777ae +runtime.selectgo(0xc0017c7710, 0xc0017c76b8, 0x7f429a499600?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017c7678 sp=0xc0017c7528 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002668eb0, {0xc00266d8a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0017c7740 sp=0xc0017c7678 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002668eb0, {0xc00266d8a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0017c77d8 sp=0xc0017c7740 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00266d7a0, {0xc00266d8a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0017c7820 sp=0xc0017c77d8 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc00266d7a0}, {0xc00266d8a0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0017c7868 sp=0xc0017c7820 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c58360, {0xc00266d8a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0017c78d8 sp=0xc0017c7868 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc00266d890, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0017c79a8 sp=0xc0017c78d8 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc001c58360, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0017c7a98 sp=0xc0017c79a8 pc=0x1171c5c +google.golang.org/grpc.recv(0xc00266d890, {0x7f424347ae80, 0x779c980}, 0x463c97f?, {0x0, 0x0}, {0x43f1c20, 0xc0030d2070}, 0xc00267f098?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0017c7b50 sp=0xc0017c7a98 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000b9c2d0, {0x43f1c20?, 0xc0030d2070}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0017c7d88 sp=0xc0017c7b50 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc00266d9e0, {0x43f1c20, 0xc0030d2070}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0017c7e70 sp=0xc0017c7d88 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002488b60, {0x43f1c20, 0xc0030d2070}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0017c7ea8 sp=0xc0017c7e70 pc=0x1fd94b2 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectServer).Recv(0xc002d00490) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:140 +0x66 fp=0xc0017c7ee8 sp=0xc0017c7ea8 pc=0x2004046 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc0017c7fe0 sp=0xc0017c7ee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017c7fe8 sp=0xc0017c7fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1511 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1538 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc0027996c0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cafdd8 sp=0xc002cafdb8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc002722050, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc002cafe28 sp=0xc002cafdd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc002722040) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc002cafe58 sp=0xc002cafe28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc0020ace00, 0xc0027996c0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc002caffb0 sp=0xc002cafe58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc002caffe0 sp=0xc002caffb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002caffe8 sp=0xc002caffe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 1524 [chan receive]: +runtime.gopark(0x431af0?, 0x0?, 0x16?, 0x0?, 0xc0028581e0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c1fe20 sp=0xc002c1fe00 pc=0x4777ae +runtime.chanrecv(0xc0028581e0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c1fe98 sp=0xc002c1fe20 pc=0x441325 +runtime.chanrecv1(0xc002849cb0?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c1fec0 sp=0xc002c1fe98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c1ffe0 sp=0xc002c1fec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c1ffe8 sp=0xc002c1ffe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1525 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc0028581e0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cb1e20 sp=0xc002cb1e00 pc=0x4777ae +runtime.chanrecv(0xc0028581e0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cb1e98 sp=0xc002cb1e20 pc=0x441325 +runtime.chanrecv1(0xc002849cb0?, 0x8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cb1ec0 sp=0xc002cb1e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cb1fe0 sp=0xc002cb1ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cb1fe8 sp=0xc002cb1fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1526 [select]: +runtime.gopark(0xc002cb0fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002cb0f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cb0de8 sp=0xc002cb0dc8 pc=0x4777ae +runtime.selectgo(0xc002cb0fb0, 0xc002cb0f70, 0xc000f427d0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cb0f38 sp=0xc002cb0de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002cb0fe0 sp=0xc002cb0f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cb0fe8 sp=0xc002cb0fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1478 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1527 [select]: +runtime.gopark(0xc002cb2fb0?, 0x2?, 0x0?, 0x0?, 0xc002cb2ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cb2d58 sp=0xc002cb2d38 pc=0x4777ae +runtime.selectgo(0xc002cb2fb0, 0xc002cb2ee0, 0x1381849?, 0x0, 0x2166b62?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cb2ea8 sp=0xc002cb2d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002cb2fe0 sp=0xc002cb2ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cb2fe8 sp=0xc002cb2fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1478 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1528 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d3bd60 sp=0xc002d3bd40 pc=0x4777ae +runtime.chanrecv(0xc002858780, 0xc002d3be60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d3bdd8 sp=0xc002d3bd60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d3be00 sp=0xc002d3bdd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002d3bfe0 sp=0xc002d3be00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d3bfe8 sp=0xc002d3bfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1478 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1539 [chan receive]: +runtime.gopark(0x3fa3c40?, 0xc00235d7a0?, 0x0?, 0x0?, 0xc002cc7f00?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cc7f08 sp=0xc002cc7ee8 pc=0x4777ae +runtime.chanrecv(0xc00235d7a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cc7f80 sp=0xc002cc7f08 pc=0x441325 +runtime.chanrecv1(0xc0027caa00?, 0xc0018534a0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cc7fa8 sp=0xc002cc7f80 pc=0x440f72 +github.com/nats-io/nats%2ego.(*js).subscribe.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/js.go:1817 +0x4a fp=0xc002cc7fe0 sp=0xc002cc7fa8 pc=0x1f4624a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cc7fe8 sp=0xc002cc7fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*js).subscribe in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/js.go:1816 +0x2c85 + +goroutine 1529 [select]: +runtime.gopark(0xc002d3cfb0?, 0x2?, 0x0?, 0xce?, 0xc002d3cf2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d3cda8 sp=0xc002d3cd88 pc=0x4777ae +runtime.selectgo(0xc002d3cfb0, 0xc002d3cf28, 0x0?, 0x0, 0x12a6d85?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d3cef8 sp=0xc002d3cda8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002d3cfe0 sp=0xc002d3cef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d3cfe8 sp=0xc002d3cfe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1530 [select]: +runtime.gopark(0xc002d3af90?, 0x2?, 0xb4?, 0x1b?, 0xc002d3aeac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d3ad28 sp=0xc002d3ad08 pc=0x4777ae +runtime.selectgo(0xc002d3af90, 0xc002d3aea8, 0x5d53d0?, 0x0, 0xc002847680?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d3ae78 sp=0xc002d3ad28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc002949a40) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002d3afc0 sp=0xc002d3ae78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002d3afe0 sp=0xc002d3afc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d3afe8 sp=0xc002d3afe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1531 [select]: +runtime.gopark(0xc002e13f40?, 0x2?, 0xb4?, 0x1b?, 0xc002e13ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e13d48 sp=0xc002e13d28 pc=0x4777ae +runtime.selectgo(0xc002e13f40, 0xc002e13ec8, 0xc001c96480?, 0x0, 0x136de51?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e13e98 sp=0xc002e13d48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002cba0f0}, 0xc002949ad0, 0x0, 0xc0022ef620, 0xc002caaf00) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc002e13f70 sp=0xc002e13e98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc002e13fe0 sp=0xc002e13f70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e13fe8 sp=0xc002e13fe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1532 [chan receive]: +runtime.gopark(0x70?, 0x0?, 0x70?, 0x0?, 0xc002858660?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d06e20 sp=0xc002d06e00 pc=0x4777ae +runtime.chanrecv(0xc002858660, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d06e98 sp=0xc002d06e20 pc=0x441325 +runtime.chanrecv1(0xc002ca3620?, 0x2106cfa?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d06ec0 sp=0xc002d06e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d06fe0 sp=0xc002d06ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d06fe8 sp=0xc002d06fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1533 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc002858660?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ccbe20 sp=0xc002ccbe00 pc=0x4777ae +runtime.chanrecv(0xc002858660, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002ccbe98 sp=0xc002ccbe20 pc=0x441325 +runtime.chanrecv1(0xc002ca3620?, 0x8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002ccbec0 sp=0xc002ccbe98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002ccbfe0 sp=0xc002ccbec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ccbfe8 sp=0xc002ccbfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1513 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d46cd0 sp=0xc002d46cb0 pc=0x4777ae +runtime.chanrecv(0xc002cce600, 0xc002d46f90, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d46d48 sp=0xc002d46cd0 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d46d70 sp=0xc002d46d48 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/stream.(*agentStreamExtensionServerImpl).Connect(0xc001e55380, {0x5b508a0, 0xc002d010a0}) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/stream/plugin_agent.go:166 +0x1065 fp=0xc002d47370 sp=0xc002d46d70 pc=0x261cb65 +github.com/rancher/opni/pkg/apis/stream/v1._Stream_Connect_Handler({0x4229120?, 0xc001e55380}, {0x5b4a6f0?, 0xc002488e40}) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:121 +0xb6 fp=0xc002d473b0 sp=0xc002d47370 pc=0x2003ed6 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x4229120, 0xc001e55380}, {0x5b4ab30, 0xc002d1c9f0}, 0xc0012e53c8?, 0x47ee1d0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002d47400 sp=0xc002d473b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x4229120, 0xc001e55380}, {0x5b4ab30, 0xc002d1c9f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002d47490 sp=0xc002d47400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4229120, 0xc001e55380}, {0x5b492f8, 0xc000b9c3c0}, 0xc0012e53c8, 0xc00268c9c0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002d477d0 sp=0xc002d47490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x4229120, 0xc001e55380}, {0x5b492f8, 0xc000b9c3c0}, 0x3f252a0?, 0xc002d00e00?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002d47848 sp=0xc002d477d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001c58c60, 0xc0027801b0, 0x770c800, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002d47db0 sp=0xc002d47848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001c58c60, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002d47f28 sp=0xc002d47db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002d47fe0 sp=0xc002d47f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d47fe8 sp=0xc002d47fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1398 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1540 [select]: +runtime.gopark(0xc002daff70?, 0x2?, 0x1?, 0x0?, 0xc002dafe7c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002dafce0 sp=0xc002dafcc0 pc=0x4777ae +runtime.selectgo(0xc002daff70, 0xc002dafe78, 0xc0027ac930?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002dafe30 sp=0xc002dafce0 pc=0x48916b +github.com/rancher/opni/pkg/storage/jetstream.(*JetStreamStore).WatchCluster.func1() + /home/ec2-user/opni/pkg/storage/jetstream/cluster_store.go:137 +0x46c fp=0xc002daffe0 sp=0xc002dafe30 pc=0x249cecc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002daffe8 sp=0xc002daffe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage/jetstream.(*JetStreamStore).WatchCluster in goroutine 1174 + /home/ec2-user/opni/pkg/storage/jetstream/cluster_store.go:110 +0x31d + +goroutine 1534 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x8e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d38e20 sp=0xc002d38e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d38e98 sp=0xc002d38e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d38ec0 sp=0xc002d38e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d38fe0 sp=0xc002d38ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d38fe8 sp=0xc002d38fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1541 [select]: +runtime.gopark(0xc002cc9f80?, 0x2?, 0x0?, 0x0?, 0xc002cc9ec4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cc9d40 sp=0xc002cc9d20 pc=0x4777ae +runtime.selectgo(0xc002cc9f80, 0xc002cc9ec0, 0x4b0571?, 0x0, 0x5d8cc5?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cc9e90 sp=0xc002cc9d40 pc=0x48916b +github.com/rancher/opni/pkg/storage.(*watchContext[...]).watch(0x0) + /home/ec2-user/opni/pkg/storage/context.go:44 +0x189 fp=0xc002cc9fb0 sp=0xc002cc9e90 pc=0x22c2fa9 +github.com/rancher/opni/pkg/storage.NewWatchContext[...].func1() + /home/ec2-user/opni/pkg/storage/context.go:37 +0x45 fp=0xc002cc9fe0 sp=0xc002cc9fb0 pc=0x22c18a5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cc9fe8 sp=0xc002cc9fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage.NewWatchContext[...] in goroutine 1174 + /home/ec2-user/opni/pkg/storage/context.go:37 +0x225 + +goroutine 1535 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x9e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d39e20 sp=0xc002d39e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d39e98 sp=0xc002d39e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x1174725?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d39ec0 sp=0xc002d39e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d39fe0 sp=0xc002d39ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d39fe8 sp=0xc002d39fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1536 [select]: +runtime.gopark(0xc002c39c98?, 0x2?, 0x67?, 0x6a?, 0xc002c39bb4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c39a28 sp=0xc002c39a08 pc=0x4777ae +runtime.selectgo(0xc002c39c98, 0xc002c39bb0, 0xc000bc4c00?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c39b78 sp=0xc002c39a28 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc0024e9360) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc002c39cc8 sp=0xc002c39b78 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc001c97c20, {0x5b426a8, 0xc002ca3620}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc002c39ee8 sp=0xc002c39cc8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Splice.func2() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:190 +0x6b fp=0xc002c39fe0 sp=0xc002c39ee8 pc=0x136faab +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c39fe8 sp=0xc002c39fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Splice in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:189 +0x61a + +goroutine 1542 [select]: +runtime.gopark(0xc002cc8fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002cc8f6c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cc8dd8 sp=0xc002cc8db8 pc=0x4777ae +runtime.selectgo(0xc002cc8fb0, 0xc002cc8f68, 0xc00235cf00?, 0x0, 0xc002046a80?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cc8f28 sp=0xc002cc8dd8 pc=0x48916b +context.(*cancelCtx).propagateCancel.func2() + /home/ec2-user/sdk/go1.21.0/src/context/context.go:505 +0xe5 fp=0xc002cc8fe0 sp=0xc002cc8f28 pc=0x5d5bc5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cc8fe8 sp=0xc002cc8fe0 pc=0x4ad421 +created by context.(*cancelCtx).propagateCancel in goroutine 1174 + /home/ec2-user/sdk/go1.21.0/src/context/context.go:504 +0x530 + +goroutine 1543 [select]: +runtime.gopark(0xc002ccafb0?, 0x2?, 0xb4?, 0x1b?, 0xc002ccaf74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ccade8 sp=0xc002ccadc8 pc=0x4777ae +runtime.selectgo(0xc002ccafb0, 0xc002ccaf70, 0xc00235c9c0?, 0x0, 0xc00235ca20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ccaf38 sp=0xc002ccade8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002ccafe0 sp=0xc002ccaf38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ccafe8 sp=0xc002ccafe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1174 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1544 [select]: +runtime.gopark(0xc002db9fb0?, 0x2?, 0x0?, 0x0?, 0xc002db9ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002db9d58 sp=0xc002db9d38 pc=0x4777ae +runtime.selectgo(0xc002db9fb0, 0xc002db9ee0, 0x448855?, 0x0, 0x4166b20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002db9ea8 sp=0xc002db9d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002db9fe0 sp=0xc002db9ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002db9fe8 sp=0xc002db9fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1174 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1545 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d86d60 sp=0xc002d86d40 pc=0x4777ae +runtime.chanrecv(0xc00235d740, 0xc002d86e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d86dd8 sp=0xc002d86d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d86e00 sp=0xc002d86dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002d86fe0 sp=0xc002d86e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d86fe8 sp=0xc002d86fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1174 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1514 [select]: +runtime.gopark(0xc002d80fb0?, 0x2?, 0x0?, 0x0?, 0xc002d80f2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d80da8 sp=0xc002d80d88 pc=0x4777ae +runtime.selectgo(0xc002d80fb0, 0xc002d80f28, 0x4b0571?, 0x0, 0x11847d3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d80ef8 sp=0xc002d80da8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002d80fe0 sp=0xc002d80ef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d80fe8 sp=0xc002d80fe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1513 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1515 [select]: +runtime.gopark(0xc002d81f90?, 0x2?, 0xff?, 0xff?, 0xc002d81eac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d81d28 sp=0xc002d81d08 pc=0x4777ae +runtime.selectgo(0xc002d81f90, 0xc002d81ea8, 0x5d5276?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d81e78 sp=0xc002d81d28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc0015ec780) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002d81fc0 sp=0xc002d81e78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002d81fe0 sp=0xc002d81fc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d81fe8 sp=0xc002d81fe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1513 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1516 [select]: +runtime.gopark(0xc002e0ff40?, 0x2?, 0xb4?, 0x1b?, 0xc002e0fecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e0fd48 sp=0xc002e0fd28 pc=0x4777ae +runtime.selectgo(0xc002e0ff40, 0xc002e0fec8, 0x4ad421?, 0x0, 0x1381849?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e0fe98 sp=0xc002e0fd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002669810}, 0xc0015ec810, 0x0, 0xc002d20de0, 0xc002d01160) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc002e0ff70 sp=0xc002e0fe98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc002e0ffe0 sp=0xc002e0ff70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e0ffe8 sp=0xc002e0ffe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1513 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1487 [select]: +runtime.gopark(0xc002d3efb0?, 0x2?, 0xb4?, 0x1b?, 0xc002d3ef74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d3ede8 sp=0xc002d3edc8 pc=0x4777ae +runtime.selectgo(0xc002d3efb0, 0xc002d3ef70, 0x0?, 0x0, 0x8d0f0a?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d3ef38 sp=0xc002d3ede8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002d3efe0 sp=0xc002d3ef38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d3efe8 sp=0xc002d3efe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1486 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1517 [chan receive]: +runtime.gopark(0xc002d1c7e0?, 0x4b05a9?, 0x50?, 0x5e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002db5e20 sp=0xc002db5e00 pc=0x4777ae +runtime.chanrecv(0xc002666720, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002db5e98 sp=0xc002db5e20 pc=0x441325 +runtime.chanrecv1(0xc002d1ca50?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002db5ec0 sp=0xc002db5e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002db5fe0 sp=0xc002db5ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002db5fe8 sp=0xc002db5fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1513 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1488 [select]: +runtime.gopark(0xc002c6dfb0?, 0x2?, 0x0?, 0x0?, 0xc002c6dee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c6dd58 sp=0xc002c6dd38 pc=0x4777ae +runtime.selectgo(0xc002c6dfb0, 0xc002c6dee0, 0x1381849?, 0x0, 0x6a55575255575268?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c6dea8 sp=0xc002c6dd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002c6dfe0 sp=0xc002c6dea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c6dfe8 sp=0xc002c6dfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1486 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1489 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d3dd60 sp=0xc002d3dd40 pc=0x4777ae +runtime.chanrecv(0xc002bf7aa0, 0xc002d3de60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d3ddd8 sp=0xc002d3dd60 pc=0x441325 +runtime.chanrecv1(0x1d?, 0xa?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d3de00 sp=0xc002d3ddd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002d3dfe0 sp=0xc002d3de00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d3dfe8 sp=0xc002d3dfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1486 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1518 [chan receive]: +runtime.gopark(0xc002d1c7e0?, 0x4b05a9?, 0x50?, 0xbe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c6be20 sp=0xc002c6be00 pc=0x4777ae +runtime.chanrecv(0xc002666720, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c6be98 sp=0xc002c6be20 pc=0x441325 +runtime.chanrecv1(0xc002d1ca50?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c6bec0 sp=0xc002c6be98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c6bfe0 sp=0xc002c6bec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c6bfe8 sp=0xc002c6bfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1513 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1537 [select]: +runtime.gopark(0xc002c3b3f8?, 0x2?, 0xc0?, 0x72?, 0xc002c3b37c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c3b1e8 sp=0xc002c3b1c8 pc=0x4777ae +runtime.selectgo(0xc002c3b3f8, 0xc002c3b378, 0x5b203e8?, 0x0, 0x2710?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c3b338 sp=0xc002c3b1e8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc002cba000, {0xc002ca3720, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc002c3b428 sp=0xc002c3b338 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002cba000, {0xc002ca3720, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc002c3b4c0 sp=0xc002c3b428 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002ca36e0, {0xc002ca3720, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002c3b508 sp=0xc002c3b4c0 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002ca36e0}, {0xc002ca3720, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002c3b550 sp=0xc002c3b508 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c97b00, {0xc002ca3720, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002c3b5c0 sp=0xc002c3b550 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002ca3710, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002c3b690 sp=0xc002c3b5c0 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc001c97b00, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002c3b780 sp=0xc002c3b690 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002ca3710, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43f1c20, 0xc0027adb90}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002c3b838 sp=0xc002c3b780 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc002ca65b0, {0x43f1c20?, 0xc0027adb90}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc002c3ba98 sp=0xc002c3b838 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001c97854?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc002c3baf8 sp=0xc002c3ba98 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001c977a0, 0xc002c3bcb0, 0xc002c3bc80) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc002c3bba0 sp=0xc002c3baf8 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001c977a0, {0x43f1c20, 0xc0027adb90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc002c3bce0 sp=0xc002c3bba0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc002571500, {0x43f1c20, 0xc0027adb90}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc002c3bea8 sp=0xc002c3bce0 pc=0x12ac5b9 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectClient).Recv(0xc002caaea0) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:71 +0x66 fp=0xc002c3bee8 sp=0xc002c3bea8 pc=0x2003b86 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc002c3bfe0 sp=0xc002c3bee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c3bfe8 sp=0xc002c3bfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1536 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1519 [chan receive]: +runtime.gopark(0xc002d1c7e0?, 0x4b05a9?, 0x50?, 0xfe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d3fe20 sp=0xc002d3fe00 pc=0x4777ae +runtime.chanrecv(0xc002666720, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d3fe98 sp=0xc002d3fe20 pc=0x441325 +runtime.chanrecv1(0xc002d1ca50?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d3fec0 sp=0xc002d3fe98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d3ffe0 sp=0xc002d3fec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d3ffe8 sp=0xc002d3ffe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1513 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1576 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc002858660?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c6ae20 sp=0xc002c6ae00 pc=0x4777ae +runtime.chanrecv(0xc002858660, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c6ae98 sp=0xc002c6ae20 pc=0x441325 +runtime.chanrecv1(0xc002ca3620?, 0x100000001?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c6aec0 sp=0xc002c6ae98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c6afe0 sp=0xc002c6aec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c6afe8 sp=0xc002c6afe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1521 [select]: +runtime.gopark(0xc002c6cfb0?, 0x2?, 0x0?, 0xce?, 0xc002c6cf2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c6cda8 sp=0xc002c6cd88 pc=0x4777ae +runtime.selectgo(0xc002c6cfb0, 0xc002c6cf28, 0x1000000?, 0x0, 0x12a6d85?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c6cef8 sp=0xc002c6cda8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002c6cfe0 sp=0xc002c6cef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c6cfe8 sp=0xc002c6cfe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1554 [select]: +runtime.gopark(0xc002c69f90?, 0x2?, 0xff?, 0xff?, 0xc002c69eac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c69d28 sp=0xc002c69d08 pc=0x4777ae +runtime.selectgo(0xc002c69f90, 0xc002c69ea8, 0x4b0571?, 0x0, 0x4b0571?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c69e78 sp=0xc002c69d28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc0015ecfc0) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002c69fc0 sp=0xc002c69e78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002c69fe0 sp=0xc002c69fc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c69fe8 sp=0xc002c69fe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1555 [select]: +runtime.gopark(0xc002e3bf40?, 0x2?, 0xb4?, 0x1b?, 0xc002e3becc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e3bd48 sp=0xc002e3bd28 pc=0x4777ae +runtime.selectgo(0xc002e3bf40, 0xc002e3bec8, 0x4ad421?, 0x0, 0x1381849?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e3be98 sp=0xc002e3bd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002669ae0}, 0xc0015ed050, 0x0, 0xc002d212c0, 0xc002d01290) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc002e3bf70 sp=0xc002e3be98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc002e3bfe0 sp=0xc002e3bf70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e3bfe8 sp=0xc002e3bfe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1556 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc00235d560?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002db7e20 sp=0xc002db7e00 pc=0x4777ae +runtime.chanrecv(0xc00235d560, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002db7e98 sp=0xc002db7e20 pc=0x441325 +runtime.chanrecv1(0xc0027cd020?, 0x5b058a8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002db7ec0 sp=0xc002db7e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002db7fe0 sp=0xc002db7ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002db7fe8 sp=0xc002db7fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1557 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc00235d560?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002db8e20 sp=0xc002db8e00 pc=0x4777ae +runtime.chanrecv(0xc00235d560, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002db8e98 sp=0xc002db8e20 pc=0x441325 +runtime.chanrecv1(0xc0027cd020?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002db8ec0 sp=0xc002db8e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002db8fe0 sp=0xc002db8ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002db8fe8 sp=0xc002db8fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1558 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x2e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d82e20 sp=0xc002d82e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d82e98 sp=0xc002d82e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0xc002009ee0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d82ec0 sp=0xc002d82e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d82fe0 sp=0xc002d82ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d82fe8 sp=0xc002d82fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1559 [select]: +runtime.gopark(0xc002c37c98?, 0x2?, 0x67?, 0x6a?, 0xc002c37bb4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c37a28 sp=0xc002c37a08 pc=0x4777ae +runtime.selectgo(0xc002c37c98, 0xc002c37bb0, 0xc002c37bc0?, 0x0, 0x43f1c20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c37b78 sp=0xc002c37a28 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc002651220) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc002c37cc8 sp=0xc002c37b78 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc001c59320, {0x5b426a8, 0xc0027cd020}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc002c37ee8 sp=0xc002c37cc8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Splice.func2() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:190 +0x6b fp=0xc002c37fe0 sp=0xc002c37ee8 pc=0x136faab +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c37fe8 sp=0xc002c37fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Splice in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:189 +0x61a + +goroutine 1546 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010ccd98 sp=0xc0010ccd78 pc=0x4777ae +runtime.chanrecv(0xc002e0af60, 0xc0010ccfb0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0010cce10 sp=0xc0010ccd98 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0010cce38 sp=0xc0010cce10 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/stream.(*gatewayStreamExtensionServerImpl).Connect(0xc001e7f570, {0x5b508a0, 0xc002c7f110}) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/stream/plugin_gateway.go:157 +0xe51 fp=0xc0010cd370 sp=0xc0010cce38 pc=0x261f931 +github.com/rancher/opni/pkg/apis/stream/v1._Stream_Connect_Handler({0x4229080?, 0xc001e7f570}, {0x5b4a6f0?, 0xc002bf9680}) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:121 +0xb6 fp=0xc0010cd3b0 sp=0xc0010cd370 pc=0x2003ed6 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x4229080, 0xc001e7f570}, {0x5b4ab30, 0xc002c73c80}, 0xc001e472c0?, 0x47ee1d0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0010cd400 sp=0xc0010cd3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x4229080, 0xc001e7f570}, {0x5b4ab30, 0xc002c73c80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0010cd490 sp=0xc0010cd400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4229080, 0xc001e7f570}, {0x5b492f8, 0xc0016710e0}, 0xc001e472c0, 0xc0027a6c80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0010cd7d0 sp=0xc0010cd490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x4229080, 0xc001e7f570}, {0x5b492f8, 0xc0016710e0}, 0x3f252a0?, 0xc002c7eff0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0010cd848 sp=0xc0010cd7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0028a0c60, 0xc001267c80, 0x770c800, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0010cddb0 sp=0xc0010cd848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0028a0c60, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0010cdf28 sp=0xc0010cddb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0010cdfe0 sp=0xc0010cdf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010cdfe8 sp=0xc0010cdfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1547 [select]: +runtime.gopark(0xc002dd32e0?, 0x4?, 0xb4?, 0x1b?, 0xc002dd3050?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002dd2e70 sp=0xc002dd2e50 pc=0x4777ae +runtime.selectgo(0xc002dd32e0, 0xc002dd3048, 0x24?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002dd2fc0 sp=0xc002dd2e70 pc=0x48916b +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).SyncConfig(0xc00070a000, {0x5b50438?, 0xc002db02a0}) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/admin.go:301 +0xb36 fp=0xc002dd3370 sp=0xc002dd2fc0 pc=0x395bbd6 +github.com/rancher/opni/plugins/alerting/apis/alertops._ConfigReconciler_SyncConfig_Handler({0x458d420?, 0xc00070a000}, {0x5b4a6f0?, 0xc002d9c200}) + /home/ec2-user/opni/plugins/alerting/apis/alertops/alertops_grpc.pb.go:468 +0xb6 fp=0xc002dd33b0 sp=0xc002dd3370 pc=0x2610b36 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x458d420, 0xc00070a000}, {0x5b4ab30, 0xc0027cd680}, 0xc00149fd10?, 0x47ec758) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002dd3400 sp=0xc002dd33b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x458d420, 0xc00070a000}, {0x5b4ab30, 0xc0027cd680}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002dd3490 sp=0xc002dd3400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x458d420, 0xc00070a000}, {0x5b492f8, 0xc00278a2d0}, 0xc00149fd10, 0xc0027225c0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002dd37d0 sp=0xc002dd3490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x458d420, 0xc00070a000}, {0x5b492f8, 0xc00278a2d0}, 0x3f252a0?, 0xc002db01d0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002dd3848 sp=0xc002dd37d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0028a0fc0, 0xc001267890, 0x770c6c0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002dd3db0 sp=0xc002dd3848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0028a0fc0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002dd3f28 sp=0xc002dd3db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002dd3fe0 sp=0xc002dd3f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002dd3fe8 sp=0xc002dd3fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1570 [semacquire]: +runtime.gopark(0x473391?, 0xc002c7b238?, 0x60?, 0x5b?, 0x448485?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c7b1c8 sp=0xc002c7b1a8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +runtime.semacquire1(0xc002d6e310, 0x58?, 0x1, 0x0, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:160 +0x218 fp=0xc002c7b230 sp=0xc002c7b1c8 pc=0x48a478 +sync.runtime_Semacquire(0xc002d6e310?) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:62 +0x25 fp=0xc002c7b268 sp=0xc002c7b230 pc=0x4a8fc5 +sync.(*WaitGroup).Wait(0xc002d6e308) + /home/ec2-user/sdk/go1.21.0/src/sync/waitgroup.go:116 +0xa5 fp=0xc002c7b298 sp=0xc002c7b268 pc=0x4c06e5 +golang.org/x/sync/errgroup.(*Group).Wait(0xc002d6e300) + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:53 +0x38 fp=0xc002c7b2c8 sp=0xc002c7b298 pc=0x20071d8 +github.com/rancher/opni/pkg/management.NewServer.unknownServiceHandler.func3({0xc000ef57c0?, 0x5b426a8?}, {0x5b4ab30?, 0xc002c721b0}) + /home/ec2-user/opni/pkg/management/extensions.go:567 +0xa85 fp=0xc002c7b508 sp=0xc002c7b2c8 pc=0x231d105 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x0, 0x0}, {0x5b492f8, 0xc001670f00}, 0xc001e467e0, 0xc00158a890) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002c7b848 sp=0xc002c7b508 pc=0x12b0e02 +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c1e0, {0x5b54e00, 0xc002c26d00}, 0xc001d8b440, 0x0, 0xc001ded2a0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002c7bdb0 sp=0xc002c7b848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c1e0, {0x5b54e00, 0xc002c26d00}, 0xc001d8b440, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1747 +0xf15 fp=0xc002c7bf28 sp=0xc002c7bdb0 pc=0x11846f5 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002c7bfe0 sp=0xc002c7bf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c7bfe8 sp=0xc002c7bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1484 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1586 [select]: +runtime.gopark(0xc002d31aa8?, 0x2?, 0x67?, 0x6a?, 0xc002d319c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d31838 sp=0xc002d31818 pc=0x4777ae +runtime.selectgo(0xc002d31aa8, 0xc002d319c0, 0xc002d319d0?, 0x0, 0x43f1c20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d31988 sp=0xc002d31838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc002651180) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc002d31ad8 sp=0xc002d31988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc001c58d80, {0x5b426a8, 0xc002d1ca50}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc002d31cf8 sp=0xc002d31ad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc002d31fe0 sp=0xc002d31cf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d31fe8 sp=0xc002d31fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 1520 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 1571 [select]: +runtime.gopark(0xc002cc4fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002cc4f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cc4de8 sp=0xc002cc4dc8 pc=0x4777ae +runtime.selectgo(0xc002cc4fb0, 0xc002cc4f70, 0x0?, 0x0, 0xc002000000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cc4f38 sp=0xc002cc4de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002cc4fe0 sp=0xc002cc4f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cc4fe8 sp=0xc002cc4fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1570 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1572 [select]: +runtime.gopark(0xc002cc5fb0?, 0x2?, 0x0?, 0x0?, 0xc002cc5ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cc5d58 sp=0xc002cc5d38 pc=0x4777ae +runtime.selectgo(0xc002cc5fb0, 0xc002cc5ee0, 0x136de51?, 0x0, 0x2005f24?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cc5ea8 sp=0xc002cc5d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002cc5fe0 sp=0xc002cc5ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cc5fe8 sp=0xc002cc5fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1570 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1573 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cc6d60 sp=0xc002cc6d40 pc=0x4777ae +runtime.chanrecv(0xc002bf7e00, 0xc002cc6e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cc6dd8 sp=0xc002cc6d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cc6e00 sp=0xc002cc6dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002cc6fe0 sp=0xc002cc6e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cc6fe8 sp=0xc002cc6fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1570 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1574 [select]: +runtime.gopark(0xc002f392e0?, 0x4?, 0xb4?, 0x1b?, 0xc002f39050?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002f38e70 sp=0xc002f38e50 pc=0x4777ae +runtime.selectgo(0xc002f392e0, 0xc002f39048, 0x24?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002f38fc0 sp=0xc002f38e70 pc=0x48916b +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).SyncConfig(0xc00070a000, {0x5b50438?, 0xc002c7e0c0}) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/admin.go:301 +0xb36 fp=0xc002f39370 sp=0xc002f38fc0 pc=0x395bbd6 +github.com/rancher/opni/plugins/alerting/apis/alertops._ConfigReconciler_SyncConfig_Handler({0x458d420?, 0xc00070a000}, {0x5b4a6f0?, 0xc002bf92c0}) + /home/ec2-user/opni/plugins/alerting/apis/alertops/alertops_grpc.pb.go:468 +0xb6 fp=0xc002f393b0 sp=0xc002f39370 pc=0x2610b36 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x458d420, 0xc00070a000}, {0x5b4ab30, 0xc002c72870}, 0xc001e46990?, 0x47ec758) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002f39400 sp=0xc002f393b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x458d420, 0xc00070a000}, {0x5b4ab30, 0xc002c72870}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002f39490 sp=0xc002f39400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x458d420, 0xc00070a000}, {0x5b492f8, 0xc001670ff0}, 0xc001e46990, 0xc002852540) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002f397d0 sp=0xc002f39490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x458d420, 0xc00070a000}, {0x5b492f8, 0xc001670ff0}, 0x3f252a0?, 0xc0026f3ab0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002f39848 sp=0xc002f397d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001d8bb00, 0xc001267890, 0x770c6c0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002f39db0 sp=0xc002f39848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001d8bb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002f39f28 sp=0xc002f39db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002f39fe0 sp=0xc002f39f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002f39fe8 sp=0xc002f39fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1575 [select]: +runtime.gopark(0xc002c11710?, 0x2?, 0x0?, 0x0?, 0xc002c116bc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c11528 sp=0xc002c11508 pc=0x4777ae +runtime.selectgo(0xc002c11710, 0xc002c116b8, 0x7f429a49fc00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c11678 sp=0xc002c11528 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002669630, {0xc002d1c8b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002c11740 sp=0xc002c11678 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002669630, {0xc002d1c8b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002c117d8 sp=0xc002c11740 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002d1c810, {0xc002d1c8b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002c11820 sp=0xc002c117d8 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002d1c810}, {0xc002d1c8b0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002c11868 sp=0xc002c11820 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c58c60, {0xc002d1c8b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002c118d8 sp=0xc002c11868 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002d1c8a0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002c119a8 sp=0xc002c118d8 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc001c58c60, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002c11a98 sp=0xc002c119a8 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002d1c8a0, {0x7f424347ae80, 0x779c980}, 0x40bed80?, {0x0, 0x0}, {0x43f1c20, 0xc00301cd20}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002c11b50 sp=0xc002c11a98 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000b9c3c0, {0x43f1c20?, 0xc00301cd20}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002c11d88 sp=0xc002c11b50 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002d1c9f0, {0x43f1c20, 0xc00301cd20}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002c11e70 sp=0xc002c11d88 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002488e40, {0x43f1c20, 0xc00301cd20}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002c11ea8 sp=0xc002c11e70 pc=0x1fd94b2 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectServer).Recv(0xc002d010a0) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:140 +0x66 fp=0xc002c11ee8 sp=0xc002c11ea8 pc=0x2004046 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc002c11fe0 sp=0xc002c11ee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c11fe8 sp=0xc002c11fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1586 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1577 [chan receive]: +runtime.gopark(0x70?, 0x0?, 0x70?, 0x0?, 0xc002858660?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c20e20 sp=0xc002c20e00 pc=0x4777ae +runtime.chanrecv(0xc002858660, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c20e98 sp=0xc002c20e20 pc=0x441325 +runtime.chanrecv1(0xc002ca3620?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c20ec0 sp=0xc002c20e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c20fe0 sp=0xc002c20ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c20fe8 sp=0xc002c20fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1578 [select]: +runtime.gopark(0xc002e21aa8?, 0x2?, 0x67?, 0x6a?, 0xc002e219c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e21838 sp=0xc002e21818 pc=0x4777ae +runtime.selectgo(0xc002e21aa8, 0xc002e219c0, 0xc002e219d0?, 0x0, 0x43f1c20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e21988 sp=0xc002e21838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc0024e9180) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc002e21ad8 sp=0xc002e21988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc001c6b9e0, {0x5b426a8, 0xc002849020}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc002e21cf8 sp=0xc002e21ad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc002e21fe0 sp=0xc002e21cf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e21fe8 sp=0xc002e21fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 1579 [select]: +runtime.gopark(0xc002d2b3c0?, 0x2?, 0xc0?, 0x72?, 0xc002d2b344?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d2b1b0 sp=0xc002d2b190 pc=0x4777ae +runtime.selectgo(0xc002d2b3c0, 0xc002d2b340, 0x5b203e8?, 0x0, 0x2710?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d2b300 sp=0xc002d2b1b0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc002c2e730, {0xc002c30760, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc002d2b3f0 sp=0xc002d2b300 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002c2e730, {0xc002c30760, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc002d2b488 sp=0xc002d2b3f0 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002c30720, {0xc002c30760, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002d2b4d0 sp=0xc002d2b488 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002c30720}, {0xc002c30760, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002d2b518 sp=0xc002d2b4d0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d8ac60, {0xc002c30760, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002d2b588 sp=0xc002d2b518 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002c30750, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002d2b658 sp=0xc002d2b588 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc001d8ac60, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002d2b748 sp=0xc002d2b658 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002c30750, {0x7f424347ae80, 0x779c980}, 0xc002852540?, {0x0, 0x0}, {0x43f1c20, 0xc0027adf10}, 0x5b492f8?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002d2b800 sp=0xc002d2b748 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc002c29380, {0x43f1c20?, 0xc0027adf10}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc002d2ba60 sp=0xc002d2b800 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001d8aad4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc002d2bac0 sp=0xc002d2ba60 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001d8aa20, 0xc002d2bc78, 0xc002d2bc48) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc002d2bb68 sp=0xc002d2bac0 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8aa20, {0x43f1c20, 0xc0027adf10}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc002d2bca8 sp=0xc002d2bb68 pc=0x119077e +github.com/rancher/opni/pkg/util/streams.(*ClientStreamWithContext).RecvMsg(0xc002bf8d40, {0x43f1c20, 0xc0027adf10}) + /home/ec2-user/opni/pkg/util/streams/streams.go:53 +0x52 fp=0xc002d2bce0 sp=0xc002d2bca8 pc=0x1fd8c72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc0021e7380, {0x43f1c20, 0xc0027adf10}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc002d2bea8 sp=0xc002d2bce0 pc=0x12ac5b9 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectClient).Recv(0xc002577a70) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:71 +0x66 fp=0xc002d2bee8 sp=0xc002d2bea8 pc=0x2003b86 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc002d2bfe0 sp=0xc002d2bee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d2bfe8 sp=0xc002d2bfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1578 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1560 [select]: +runtime.gopark(0xc002d653f8?, 0x2?, 0xc0?, 0x72?, 0xc002d6537c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d651e8 sp=0xc002d651c8 pc=0x4777ae +runtime.selectgo(0xc002d653f8, 0xc002d65378, 0x5b203e8?, 0x0, 0x2710?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d65338 sp=0xc002d651e8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0027cac80, {0xc0027cd120, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc002d65428 sp=0xc002d65338 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0027cac80, {0xc0027cd120, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc002d654c0 sp=0xc002d65428 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0027cd0e0, {0xc0027cd120, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002d65508 sp=0xc002d654c0 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0027cd0e0}, {0xc0027cd120, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002d65550 sp=0xc002d65508 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0028a0900, {0xc0027cd120, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002d655c0 sp=0xc002d65550 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0027cd110, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002d65690 sp=0xc002d655c0 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc0028a0900, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002d65780 sp=0xc002d65690 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0027cd110, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43f1c20, 0xc00310e7e0}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002d65838 sp=0xc002d65780 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc002799860, {0x43f1c20?, 0xc00310e7e0}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc002d65a98 sp=0xc002d65838 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0028a0654?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc002d65af8 sp=0xc002d65a98 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc0028a05a0, 0xc002d65cb0, 0xc002d65c80) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc002d65ba0 sp=0xc002d65af8 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0028a05a0, {0x43f1c20, 0xc00310e7e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc002d65ce0 sp=0xc002d65ba0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc0027222c0, {0x43f1c20, 0xc00310e7e0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc002d65ea8 sp=0xc002d65ce0 pc=0x12ac5b9 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectClient).Recv(0xc002857d80) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:71 +0x66 fp=0xc002d65ee8 sp=0xc002d65ea8 pc=0x2003b86 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc002d65fe0 sp=0xc002d65ee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d65fe8 sp=0xc002d65fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1559 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1580 [select]: +runtime.gopark(0xc002e00fb0?, 0x2?, 0x0?, 0x0?, 0xc002e00f2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e00da8 sp=0xc002e00d88 pc=0x4777ae +runtime.selectgo(0xc002e00fb0, 0xc002e00f28, 0x4b0571?, 0x0, 0x11847d3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e00ef8 sp=0xc002e00da8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002e00fe0 sp=0xc002e00ef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e00fe8 sp=0xc002e00fe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1546 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1581 [select]: +runtime.gopark(0xc002e01f90?, 0x2?, 0xff?, 0xff?, 0xc002e01eac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e01d28 sp=0xc002e01d08 pc=0x4777ae +runtime.selectgo(0xc002e01f90, 0xc002e01ea8, 0x4b0571?, 0x0, 0x448855?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e01e78 sp=0xc002e01d28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc000f25d40) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002e01fc0 sp=0xc002e01e78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002e01fe0 sp=0xc002e01fc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e01fe8 sp=0xc002e01fe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1546 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1582 [select]: +runtime.gopark(0xc002e3ff40?, 0x2?, 0xb4?, 0x1b?, 0xc002e3fecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e3fd48 sp=0xc002e3fd28 pc=0x4777ae +runtime.selectgo(0xc002e3ff40, 0xc002e3fec8, 0x4ad421?, 0x0, 0x1381849?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e3fe98 sp=0xc002e3fd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002e14690}, 0xc000f25dd0, 0x0, 0xc002e0ad20, 0xc002c7f1b0) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc002e3ff70 sp=0xc002e3fe98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc002e3ffe0 sp=0xc002e3ff70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e3ffe8 sp=0xc002e3ffe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1546 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1583 [chan receive]: +runtime.gopark(0xc0027cd320?, 0x4b05a9?, 0x50?, 0x3e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e03e20 sp=0xc002e03e00 pc=0x4777ae +runtime.chanrecv(0xc00235d800, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e03e98 sp=0xc002e03e20 pc=0x441325 +runtime.chanrecv1(0xc002c73ce0?, 0x2005f24?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e03ec0 sp=0xc002e03e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e03fe0 sp=0xc002e03ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e03fe8 sp=0xc002e03fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1546 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1584 [chan receive]: +runtime.gopark(0xc0027cd320?, 0x4b05a9?, 0x50?, 0xbe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cdbe20 sp=0xc002cdbe00 pc=0x4777ae +runtime.chanrecv(0xc00235d800, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cdbe98 sp=0xc002cdbe20 pc=0x441325 +runtime.chanrecv1(0xc002c73ce0?, 0x6a55575255575268?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cdbec0 sp=0xc002cdbe98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cdbfe0 sp=0xc002cdbec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cdbfe8 sp=0xc002cdbfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1546 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1585 [chan receive]: +runtime.gopark(0xc0027cd320?, 0x4b05a9?, 0x50?, 0x4e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e04e20 sp=0xc002e04e00 pc=0x4777ae +runtime.chanrecv(0xc00235d800, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e04e98 sp=0xc002e04e20 pc=0x441325 +runtime.chanrecv1(0xc002c73ce0?, 0x4166b20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e04ec0 sp=0xc002e04e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e04fe0 sp=0xc002e04ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e04fe8 sp=0xc002e04fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1546 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1602 [chan receive]: +runtime.gopark(0xc0027cd320?, 0x4b05a9?, 0x50?, 0x7e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e07e20 sp=0xc002e07e00 pc=0x4777ae +runtime.chanrecv(0xc00235d800, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e07e98 sp=0xc002e07e20 pc=0x441325 +runtime.chanrecv1(0xc002c73ce0?, 0x1102359?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e07ec0 sp=0xc002e07e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e07fe0 sp=0xc002e07ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e07fe8 sp=0xc002e07fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1546 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1603 [select]: +runtime.gopark(0xc002e1baa8?, 0x2?, 0x67?, 0x6a?, 0xc002e1b9c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e1b838 sp=0xc002e1b818 pc=0x4777ae +runtime.selectgo(0xc002e1baa8, 0xc002e1b9c0, 0xc002ccea20?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e1b988 sp=0xc002e1b838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc002c2ad20) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc002e1bad8 sp=0xc002e1b988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc00285a120, {0x5b426a8, 0xc002c73ce0}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc002e1bcf8 sp=0xc002e1bad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc002e1bfe0 sp=0xc002e1bcf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e1bfe8 sp=0xc002e1bfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 1546 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 1561 [select]: +runtime.gopark(0xc002e1d710?, 0x2?, 0x0?, 0x0?, 0xc002e1d6bc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e1d528 sp=0xc002e1d508 pc=0x4777ae +runtime.selectgo(0xc002e1d710, 0xc002e1d6b8, 0x7f429a4a5300?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e1d678 sp=0xc002e1d528 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0027caeb0, {0xc002c73b40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002e1d740 sp=0xc002e1d678 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0027caeb0, {0xc002c73b40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002e1d7d8 sp=0xc002e1d740 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0027cd350, {0xc002c73b40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002e1d820 sp=0xc002e1d7d8 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0027cd350}, {0xc002c73b40, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002e1d868 sp=0xc002e1d820 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0028a0c60, {0xc002c73b40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002e1d8d8 sp=0xc002e1d868 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002c73b30, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002e1d9a8 sp=0xc002e1d8d8 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc0028a0c60, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002e1da98 sp=0xc002e1d9a8 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002c73b30, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43f1c20, 0xc0027acfc0}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002e1db50 sp=0xc002e1da98 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc0016710e0, {0x43f1c20?, 0xc0027acfc0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002e1dd88 sp=0xc002e1db50 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002c73c80, {0x43f1c20, 0xc0027acfc0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002e1de70 sp=0xc002e1dd88 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002bf9680, {0x43f1c20, 0xc0027acfc0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002e1dea8 sp=0xc002e1de70 pc=0x1fd94b2 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectServer).Recv(0xc002c7f110) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:140 +0x66 fp=0xc002e1dee8 sp=0xc002e1dea8 pc=0x2004046 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc002e1dfe0 sp=0xc002e1dee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e1dfe8 sp=0xc002e1dfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1603 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1604 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc00235d560?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002db6e20 sp=0xc002db6e00 pc=0x4777ae +runtime.chanrecv(0xc00235d560, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002db6e98 sp=0xc002db6e20 pc=0x441325 +runtime.chanrecv1(0xc0027cd020?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002db6ec0 sp=0xc002db6e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002db6fe0 sp=0xc002db6ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002db6fe8 sp=0xc002db6fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1605 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc00235d560?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cd7e20 sp=0xc002cd7e00 pc=0x4777ae +runtime.chanrecv(0xc00235d560, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cd7e98 sp=0xc002cd7e20 pc=0x441325 +runtime.chanrecv1(0xc0027cd020?, 0x1102359?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cd7ec0 sp=0xc002cd7e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cd7fe0 sp=0xc002cd7ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cd7fe8 sp=0xc002cd7fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1606 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc00235d560?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e05e20 sp=0xc002e05e00 pc=0x4777ae +runtime.chanrecv(0xc00235d560, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e05e98 sp=0xc002e05e20 pc=0x441325 +runtime.chanrecv1(0xc0027cd020?, 0x8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e05ec0 sp=0xc002e05e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e05fe0 sp=0xc002e05ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e05fe8 sp=0xc002e05fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1607 [select]: +runtime.gopark(0xc002cd8fb0?, 0x2?, 0xff?, 0xff?, 0xc002cd8f6c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cd8dd8 sp=0xc002cd8db8 pc=0x4777ae +runtime.selectgo(0xc002cd8fb0, 0xc002cd8f68, 0x202020200a353620?, 0x0, 0x20200a3936202d20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cd8f28 sp=0xc002cd8dd8 pc=0x48916b +context.(*cancelCtx).propagateCancel.func2() + /home/ec2-user/sdk/go1.21.0/src/context/context.go:505 +0xe5 fp=0xc002cd8fe0 sp=0xc002cd8f28 pc=0x5d5bc5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cd8fe8 sp=0xc002cd8fe0 pc=0x4ad421 +created by context.(*cancelCtx).propagateCancel in goroutine 1174 + /home/ec2-user/sdk/go1.21.0/src/context/context.go:504 +0x530 + +goroutine 1608 [select]: +runtime.gopark(0xc002e06fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002e06f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e06de8 sp=0xc002e06dc8 pc=0x4777ae +runtime.selectgo(0xc002e06fb0, 0xc002e06f70, 0x21557a5?, 0x0, 0xc0026882c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e06f38 sp=0xc002e06de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002e06fe0 sp=0xc002e06f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e06fe8 sp=0xc002e06fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1174 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1609 [select]: +runtime.gopark(0xc002cd9fb0?, 0x2?, 0x0?, 0x0?, 0xc002cd9ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cd9d58 sp=0xc002cd9d38 pc=0x4777ae +runtime.selectgo(0xc002cd9fb0, 0xc002cd9ee0, 0x3737202d20202020?, 0x0, 0x2d202020200a3430?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cd9ea8 sp=0xc002cd9d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002cd9fe0 sp=0xc002cd9ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cd9fe8 sp=0xc002cd9fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1174 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1610 [chan receive]: +runtime.gopark(0x4b0571?, 0x12a7458?, 0xd0?, 0x2d?, 0x1377450?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e02d60 sp=0xc002e02d40 pc=0x4777ae +runtime.chanrecv(0xc002e18420, 0xc002e02e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e02dd8 sp=0xc002e02d60 pc=0x441325 +runtime.chanrecv1(0x12a7405?, 0xc002e02e38?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e02e00 sp=0xc002e02dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002e02fe0 sp=0xc002e02e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e02fe8 sp=0xc002e02fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1174 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1587 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ce8d98 sp=0xc002ce8d78 pc=0x4777ae +runtime.chanrecv(0xc002ccf740, 0xc002ce8fb0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002ce8e10 sp=0xc002ce8d98 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002ce8e38 sp=0xc002ce8e10 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/stream.(*gatewayStreamExtensionServerImpl).Connect(0xc0007d1b90, {0x5b508a0, 0xc002cab6d0}) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/stream/plugin_gateway.go:157 +0xe51 fp=0xc002ce9370 sp=0xc002ce8e38 pc=0x261f931 +github.com/rancher/opni/pkg/apis/stream/v1._Stream_Connect_Handler({0x4229080?, 0xc0007d1b90}, {0x5b4a6f0?, 0xc002cc21e0}) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:121 +0xb6 fp=0xc002ce93b0 sp=0xc002ce9370 pc=0x2003ed6 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x4229080, 0xc0007d1b90}, {0x5b4ab30, 0xc002cd24b0}, 0xc00217f5f0?, 0x47ee1d0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002ce9400 sp=0xc002ce93b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x4229080, 0xc0007d1b90}, {0x5b4ab30, 0xc002cd24b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002ce9490 sp=0xc002ce9400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4229080, 0xc0007d1b90}, {0x5b492f8, 0xc001389c20}, 0xc00217f5f0, 0xc002571f40) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002ce97d0 sp=0xc002ce9490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x4229080, 0xc0007d1b90}, {0x5b492f8, 0xc001389c20}, 0x3f252a0?, 0xc002cab5b0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002ce9848 sp=0xc002ce97d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc002906240, 0xc0010e4f90, 0x770c800, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002ce9db0 sp=0xc002ce9848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc002906240, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002ce9f28 sp=0xc002ce9db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002ce9fe0 sp=0xc002ce9f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ce9fe8 sp=0xc002ce9fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1611 [select]: +runtime.gopark(0xc002db2fb0?, 0x2?, 0x0?, 0x2e?, 0xc002db2f2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002db2da8 sp=0xc002db2d88 pc=0x4777ae +runtime.selectgo(0xc002db2fb0, 0xc002db2f28, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002db2ef8 sp=0xc002db2da8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002db2fe0 sp=0xc002db2ef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002db2fe8 sp=0xc002db2fe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1612 [select]: +runtime.gopark(0xc002db3f90?, 0x2?, 0xff?, 0xff?, 0xc002db3eac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002db3d28 sp=0xc002db3d08 pc=0x4777ae +runtime.selectgo(0xc002db3f90, 0xc002db3ea8, 0x3736202d20202020?, 0x0, 0x202d202020200a30?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002db3e78 sp=0xc002db3d28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc0011ea240) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002db3fc0 sp=0xc002db3e78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002db3fe0 sp=0xc002db3fc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002db3fe8 sp=0xc002db3fe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1613 [select]: +runtime.gopark(0xc002e59f40?, 0x2?, 0xb4?, 0x1b?, 0xc002e59ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e59d48 sp=0xc002e59d28 pc=0x4777ae +runtime.selectgo(0xc002e59f40, 0xc002e59ec8, 0x4ad421?, 0x0, 0x1381849?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e59e98 sp=0xc002e59d48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002e14f50}, 0xc0011ea2d0, 0x0, 0xc002e0b9e0, 0xc002c7f940) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc002e59f70 sp=0xc002e59e98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc002e59fe0 sp=0xc002e59f70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e59fe8 sp=0xc002e59fe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1614 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc002858a80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002eb8e20 sp=0xc002eb8e00 pc=0x4777ae +runtime.chanrecv(0xc002858a80, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002eb8e98 sp=0xc002eb8e20 pc=0x441325 +runtime.chanrecv1(0xc002e35050?, 0x1102359?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002eb8ec0 sp=0xc002eb8e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002eb8fe0 sp=0xc002eb8ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002eb8fe8 sp=0xc002eb8fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1615 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc002858a80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002eb9e20 sp=0xc002eb9e00 pc=0x4777ae +runtime.chanrecv(0xc002858a80, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002eb9e98 sp=0xc002eb9e20 pc=0x441325 +runtime.chanrecv1(0xc002e35050?, 0xc000f3e6e8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002eb9ec0 sp=0xc002eb9e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002eb9fe0 sp=0xc002eb9ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002eb9fe8 sp=0xc002eb9fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1616 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x7e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002eb7e20 sp=0xc002eb7e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002eb7e98 sp=0xc002eb7e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x6a55575255575268?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002eb7ec0 sp=0xc002eb7e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002eb7fe0 sp=0xc002eb7ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002eb7fe8 sp=0xc002eb7fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1617 [select]: +runtime.gopark(0xc002ea1c98?, 0x2?, 0x67?, 0x6a?, 0xc002ea1bb4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ea1a28 sp=0xc002ea1a08 pc=0x4777ae +runtime.selectgo(0xc002ea1c98, 0xc002ea1bb0, 0xc002ea1bc0?, 0x0, 0x43f1c20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ea1b78 sp=0xc002ea1a28 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc002c2adc0) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc002ea1cc8 sp=0xc002ea1b78 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc00285aea0, {0x5b426a8, 0xc002e35050}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc002ea1ee8 sp=0xc002ea1cc8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Splice.func2() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:190 +0x6b fp=0xc002ea1fe0 sp=0xc002ea1ee8 pc=0x136faab +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ea1fe8 sp=0xc002ea1fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Splice in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:189 +0x61a + +goroutine 1588 [select]: +runtime.gopark(0xc002cfffb0?, 0x2?, 0x0?, 0x0?, 0xc002cfff2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cffda8 sp=0xc002cffd88 pc=0x4777ae +runtime.selectgo(0xc002cfffb0, 0xc002cfff28, 0x4b0571?, 0x0, 0x11847d3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cffef8 sp=0xc002cffda8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002cfffe0 sp=0xc002cffef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cfffe8 sp=0xc002cfffe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1587 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1589 [select]: +runtime.gopark(0xc002cfdf90?, 0x2?, 0xb4?, 0x1b?, 0xc002cfdeac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cfdd28 sp=0xc002cfdd08 pc=0x4777ae +runtime.selectgo(0xc002cfdf90, 0xc002cfdea8, 0x5d53d0?, 0x0, 0xc002669810?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cfde78 sp=0xc002cfdd28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc002949c80) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002cfdfc0 sp=0xc002cfde78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002cfdfe0 sp=0xc002cfdfc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cfdfe8 sp=0xc002cfdfe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1587 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1590 [select]: +runtime.gopark(0xc00301bf40?, 0x2?, 0xb4?, 0x1b?, 0xc00301becc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00301bd48 sp=0xc00301bd28 pc=0x4777ae +runtime.selectgo(0xc00301bf40, 0xc00301bec8, 0x5d8cc5?, 0x0, 0xc002d1c990?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00301be98 sp=0xc00301bd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002cba780}, 0xc002949d10, 0x0, 0xc002ccef60, 0xc002cab960) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc00301bf70 sp=0xc00301be98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc00301bfe0 sp=0xc00301bf70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00301bfe8 sp=0xc00301bfe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1587 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1591 [chan receive]: +runtime.gopark(0xc002cd22a0?, 0x4b05a9?, 0x50?, 0x5e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cd5e20 sp=0xc002cd5e00 pc=0x4777ae +runtime.chanrecv(0xc002858ae0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cd5e98 sp=0xc002cd5e20 pc=0x441325 +runtime.chanrecv1(0xc002cd2510?, 0x448845?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cd5ec0 sp=0xc002cd5e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cd5fe0 sp=0xc002cd5ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cd5fe8 sp=0xc002cd5fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1587 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1592 [chan receive]: +runtime.gopark(0xc002cd22a0?, 0x4b05a9?, 0x50?, 0x6e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cd6e20 sp=0xc002cd6e00 pc=0x4777ae +runtime.chanrecv(0xc002858ae0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cd6e98 sp=0xc002cd6e20 pc=0x441325 +runtime.chanrecv1(0xc002cd2510?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cd6ec0 sp=0xc002cd6e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cd6fe0 sp=0xc002cd6ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cd6fe8 sp=0xc002cd6fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1587 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1593 [chan receive]: +runtime.gopark(0xc002cd22a0?, 0x4b05a9?, 0x50?, 0xbe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cfbe20 sp=0xc002cfbe00 pc=0x4777ae +runtime.chanrecv(0xc002858ae0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cfbe98 sp=0xc002cfbe20 pc=0x441325 +runtime.chanrecv1(0xc002cd2510?, 0x1174725?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cfbec0 sp=0xc002cfbe98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cfbfe0 sp=0xc002cfbec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cfbfe8 sp=0xc002cfbfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1587 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1594 [chan receive]: +runtime.gopark(0xc002cd22a0?, 0x4b05a9?, 0x50?, 0xce?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cfce20 sp=0xc002cfce00 pc=0x4777ae +runtime.chanrecv(0xc002858ae0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cfce98 sp=0xc002cfce20 pc=0x441325 +runtime.chanrecv1(0xc002cd2510?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cfcec0 sp=0xc002cfce98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cfcfe0 sp=0xc002cfcec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cfcfe8 sp=0xc002cfcfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1587 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1595 [chan receive]: +runtime.gopark(0xc002cd22a0?, 0x4b05a9?, 0x50?, 0xae?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cdae20 sp=0xc002cdae00 pc=0x4777ae +runtime.chanrecv(0xc002858ae0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cdae98 sp=0xc002cdae20 pc=0x441325 +runtime.chanrecv1(0xc002cd2510?, 0x8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cdaec0 sp=0xc002cdae98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cdafe0 sp=0xc002cdaec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cdafe8 sp=0xc002cdafe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1587 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1596 [select]: +runtime.gopark(0xc002d2daa8?, 0x2?, 0x67?, 0x6a?, 0xc002d2d9c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d2d838 sp=0xc002d2d818 pc=0x4777ae +runtime.selectgo(0xc002d2daa8, 0xc002d2d9c0, 0xc002ccf980?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d2d988 sp=0xc002d2d838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc0024e94a0) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc002d2dad8 sp=0xc002d2d988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc002907680, {0x5b426a8, 0xc002cd2510}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc002d2dcf8 sp=0xc002d2dad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc002d2dfe0 sp=0xc002d2dcf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d2dfe8 sp=0xc002d2dfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 1587 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 1597 [select]: +runtime.gopark(0xc002d2f710?, 0x2?, 0x0?, 0x0?, 0xc002d2f6bc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d2f528 sp=0xc002d2f508 pc=0x4777ae +runtime.selectgo(0xc002d2f710, 0xc002d2f6b8, 0x7f429a4a8600?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d2f678 sp=0xc002d2f528 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002cba460, {0xc002cd2370, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002d2f740 sp=0xc002d2f678 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002cba460, {0xc002cd2370, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002d2f7d8 sp=0xc002d2f740 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002cd22d0, {0xc002cd2370, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002d2f820 sp=0xc002d2f7d8 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002cd22d0}, {0xc002cd2370, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002d2f868 sp=0xc002d2f820 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc002906240, {0xc002cd2370, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002d2f8d8 sp=0xc002d2f868 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002cd2360, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002d2f9a8 sp=0xc002d2f8d8 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc002906240, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002d2fa98 sp=0xc002d2f9a8 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002cd2360, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43f1c20, 0xc002e99d50}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002d2fb50 sp=0xc002d2fa98 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001389c20, {0x43f1c20?, 0xc002e99d50}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002d2fd88 sp=0xc002d2fb50 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002cd24b0, {0x43f1c20, 0xc002e99d50}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002d2fe70 sp=0xc002d2fd88 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002cc21e0, {0x43f1c20, 0xc002e99d50}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002d2fea8 sp=0xc002d2fe70 pc=0x1fd94b2 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectServer).Recv(0xc002cab6d0) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:140 +0x66 fp=0xc002d2fee8 sp=0xc002d2fea8 pc=0x2004046 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc002d2ffe0 sp=0xc002d2fee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d2ffe8 sp=0xc002d2ffe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1596 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1598 [select]: +runtime.gopark(0xc002d5f3f8?, 0x2?, 0xc0?, 0x72?, 0xc002d5f37c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d5f1e8 sp=0xc002d5f1c8 pc=0x4777ae +runtime.selectgo(0xc002d5f3f8, 0xc002d5f378, 0x5b203e8?, 0x0, 0x2710?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d5f338 sp=0xc002d5f1e8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc002e14e60, {0xc002e35150, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc002d5f428 sp=0xc002d5f338 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002e14e60, {0xc002e35150, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc002d5f4c0 sp=0xc002d5f428 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002e35110, {0xc002e35150, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002d5f508 sp=0xc002d5f4c0 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002e35110}, {0xc002e35150, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002d5f550 sp=0xc002d5f508 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00285ad80, {0xc002e35150, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002d5f5c0 sp=0xc002d5f550 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002e35140, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002d5f690 sp=0xc002d5f5c0 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc00285ad80, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002d5f780 sp=0xc002d5f690 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002e35140, {0x7f424347ae80, 0x779c980}, 0x5b16300?, {0x0, 0x0}, {0x43f1c20, 0xc002e6d8f0}, 0x5?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002d5f838 sp=0xc002d5f780 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc002e2a680, {0x43f1c20?, 0xc002e6d8f0}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc002d5fa98 sp=0xc002d5f838 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00285aad4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc002d5faf8 sp=0xc002d5fa98 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00285aa20, 0xc002d5fcb0, 0xc002d5fc80) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc002d5fba0 sp=0xc002d5faf8 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00285aa20, {0x43f1c20, 0xc002e6d8f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc002d5fce0 sp=0xc002d5fba0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc0027a7900, {0x43f1c20, 0xc002e6d8f0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc002d5fea8 sp=0xc002d5fce0 pc=0x12ac5b9 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectClient).Recv(0xc002c7f8d0) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:71 +0x66 fp=0xc002d5fee8 sp=0xc002d5fea8 pc=0x2003b86 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc002d5ffe0 sp=0xc002d5fee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d5ffe8 sp=0xc002d5ffe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1617 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1599 [chan receive]: +runtime.gopark(0x70?, 0x0?, 0x70?, 0x0?, 0xc002858a80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cfee20 sp=0xc002cfee00 pc=0x4777ae +runtime.chanrecv(0xc002858a80, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cfee98 sp=0xc002cfee20 pc=0x441325 +runtime.chanrecv1(0xc002e35050?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cfeec0 sp=0xc002cfee98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cfefe0 sp=0xc002cfeec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cfefe8 sp=0xc002cfefe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1600 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc002858a80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ebbe20 sp=0xc002ebbe00 pc=0x4777ae +runtime.chanrecv(0xc002858a80, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002ebbe98 sp=0xc002ebbe20 pc=0x441325 +runtime.chanrecv1(0xc002e35050?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002ebbec0 sp=0xc002ebbe98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002ebbfe0 sp=0xc002ebbec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ebbfe8 sp=0xc002ebbfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1601 [chan receive]: +runtime.gopark(0x70?, 0x0?, 0x70?, 0x0?, 0xc002858a80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cf9e20 sp=0xc002cf9e00 pc=0x4777ae +runtime.chanrecv(0xc002858a80, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cf9e98 sp=0xc002cf9e20 pc=0x441325 +runtime.chanrecv1(0xc002e35050?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cf9ec0 sp=0xc002cf9e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cf9fe0 sp=0xc002cf9ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cf9fe8 sp=0xc002cf9fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1618 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc002858a80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cf8e20 sp=0xc002cf8e00 pc=0x4777ae +runtime.chanrecv(0xc002858a80, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cf8e98 sp=0xc002cf8e20 pc=0x441325 +runtime.chanrecv1(0xc002e35050?, 0x2005f24?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cf8ec0 sp=0xc002cf8e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cf8fe0 sp=0xc002cf8ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cf8fe8 sp=0xc002cf8fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1619 [select]: +runtime.gopark(0xc002e9daa8?, 0x2?, 0x67?, 0x6a?, 0xc002e9d9c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e9d838 sp=0xc002e9d818 pc=0x4777ae +runtime.selectgo(0xc002e9daa8, 0xc002e9d9c0, 0xc002047b60?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e9d988 sp=0xc002e9d838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc0025ca8c0) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc002e9dad8 sp=0xc002e9d988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc0019cb8c0, {0x5b426a8, 0xc0027cc1b0}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc002e9dcf8 sp=0xc002e9dad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc002e9dfe0 sp=0xc002e9dcf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e9dfe8 sp=0xc002e9dfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 1620 [select]: +runtime.gopark(0xc002e9f6d8?, 0x2?, 0x0?, 0x0?, 0xc002e9f684?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e9f4f0 sp=0xc002e9f4d0 pc=0x4777ae +runtime.selectgo(0xc002e9f6d8, 0xc002e9f680, 0x7f429a4a7100?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e9f640 sp=0xc002e9f4f0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002c2e870, {0xc002c30a00, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002e9f708 sp=0xc002e9f640 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002c2e870, {0xc002c30a00, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002e9f7a0 sp=0xc002e9f708 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002c30990, {0xc002c30a00, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002e9f7e8 sp=0xc002e9f7a0 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002c30990}, {0xc002c30a00, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002e9f830 sp=0xc002e9f7e8 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d8ad80, {0xc002c30a00, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002e9f8a0 sp=0xc002e9f830 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002c309f0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002e9f970 sp=0xc002e9f8a0 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc001d8ad80, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002e9fa60 sp=0xc002e9f970 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002c309f0, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43f1c20, 0xc002fe0460}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002e9fb18 sp=0xc002e9fa60 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001670d20, {0x43f1c20?, 0xc002fe0460}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002e9fd50 sp=0xc002e9fb18 pc=0x119905e +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002bf8da0, {0x43f1c20, 0xc002fe0460}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002e9fd88 sp=0xc002e9fd50 pc=0x1fd94b2 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002488960, {0x43f1c20, 0xc002fe0460}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002e9fdc0 sp=0xc002e9fd88 pc=0x1fd94b2 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0027cc1e0, {0x43f1c20, 0xc002fe0460}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002e9fea8 sp=0xc002e9fdc0 pc=0x12b01b3 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectServer).Recv(0xc002857580) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:140 +0x66 fp=0xc002e9fee8 sp=0xc002e9fea8 pc=0x2004046 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc002e9ffe0 sp=0xc002e9fee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e9ffe8 sp=0xc002e9ffe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1619 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1634 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x7e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e77e20 sp=0xc002e77e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e77e98 sp=0xc002e77e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e77ec0 sp=0xc002e77e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e77fe0 sp=0xc002e77ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e77fe8 sp=0xc002e77fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1635 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x4e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002db4e20 sp=0xc002db4e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002db4e98 sp=0xc002db4e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x2005f24?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002db4ec0 sp=0xc002db4e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002db4fe0 sp=0xc002db4ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002db4fe8 sp=0xc002db4fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1636 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x4e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002eb4e20 sp=0xc002eb4e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002eb4e98 sp=0xc002eb4e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002eb4ec0 sp=0xc002eb4e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002eb4fe0 sp=0xc002eb4ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002eb4fe8 sp=0xc002eb4fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1637 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x6e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002eb6e20 sp=0xc002eb6e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002eb6e98 sp=0xc002eb6e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002eb6ec0 sp=0xc002eb6e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002eb6fe0 sp=0xc002eb6ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002eb6fe8 sp=0xc002eb6fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1638 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x5e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002eb5e20 sp=0xc002eb5e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002eb5e98 sp=0xc002eb5e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x2005f24?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002eb5ec0 sp=0xc002eb5e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002eb5fe0 sp=0xc002eb5ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002eb5fe8 sp=0xc002eb5fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1639 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x3e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e73e20 sp=0xc002e73e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e73e98 sp=0xc002e73e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x1102359?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e73ec0 sp=0xc002e73e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e73fe0 sp=0xc002e73ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e73fe8 sp=0xc002e73fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1640 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x5e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e75e20 sp=0xc002e75e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e75e98 sp=0xc002e75e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e75ec0 sp=0xc002e75e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e75fe0 sp=0xc002e75ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e75fe8 sp=0xc002e75fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1641 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x4e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e74e20 sp=0xc002e74e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e74e98 sp=0xc002e74e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x5b058a8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e74ec0 sp=0xc002e74e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e74fe0 sp=0xc002e74ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e74fe8 sp=0xc002e74fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1642 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0xfe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e7fe20 sp=0xc002e7fe00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e7fe98 sp=0xc002e7fe20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e7fec0 sp=0xc002e7fe98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e7ffe0 sp=0xc002e7fec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e7ffe8 sp=0xc002e7ffe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1643 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x6e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e76e20 sp=0xc002e76e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e76e98 sp=0xc002e76e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e76ec0 sp=0xc002e76e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e76fe0 sp=0xc002e76ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e76fe8 sp=0xc002e76fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1648 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc002fa45b0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029dfdd8 sp=0xc0029dfdb8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc002f90090, 0x3) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc0029dfe28 sp=0xc0029dfdd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc002f90080) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc0029dfe58 sp=0xc0029dfe28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc001659100, 0xc002fa45b0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc0029dffb0 sp=0xc0029dfe58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc0029dffe0 sp=0xc0029dffb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029dffe8 sp=0xc0029dffe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 1139 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 1645 [select]: +runtime.gopark(0xc002e55ee0?, 0x4?, 0x67?, 0x6a?, 0xc002e55ba0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e559b8 sp=0xc002e55998 pc=0x4777ae +runtime.selectgo(0xc002e55ee0, 0xc002e55b98, 0x0?, 0x0, 0x3f99540?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e55b08 sp=0xc002e559b8 pc=0x48916b +github.com/rancher/opni/pkg/health.(*Listener).HandleConnection(0xc001623400, {0x5b430b0, 0xc0027cce40}, {0x7f421ad8c958, 0xc002e61a40}) + /home/ec2-user/opni/pkg/health/listener.go:195 +0xd1c fp=0xc002e55f30 sp=0xc002e55b08 pc=0x206ca3c +github.com/rancher/opni/pkg/health.(*Listener).HandleAgentConnection(0xc000f1ef48?, {0x5b430b0, 0xc0027cce40}, {0x5b2bf70?, 0xc002e61a40}) + /home/ec2-user/opni/pkg/health/listener.go:257 +0x65 fp=0xc002e55f78 sp=0xc002e55f30 pc=0x206d6a5 +github.com/rancher/opni/pkg/gateway.(*multiConnectionHandler).HandleAgentConnection.func1() + /home/ec2-user/opni/pkg/gateway/grpc.go:40 +0x82 fp=0xc002e55fe0 sp=0xc002e55f78 pc=0x22b3a62 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e55fe8 sp=0xc002e55fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.(*multiConnectionHandler).HandleAgentConnection in goroutine 1644 + /home/ec2-user/opni/pkg/gateway/grpc.go:40 +0x6f + +goroutine 1646 [select]: +runtime.gopark(0xc002ebfe58?, 0x2?, 0x67?, 0x6a?, 0xc002ebfd94?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ebfbe8 sp=0xc002ebfbc8 pc=0x4777ae +runtime.selectgo(0xc002ebfe58, 0xc002ebfd90, 0xc002e4fa80?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ebfd38 sp=0xc002ebfbe8 pc=0x48916b +github.com/rancher/opni/pkg/gateway.(*SyncRequester).runPeriodicSync(0xc0017e1740, {0x5b430b0?, 0xc0027cce40}, 0xc001d6fc90?, 0x45d964b800, 0x0?) + /home/ec2-user/opni/pkg/gateway/sync.go:110 +0x1b2 fp=0xc002ebfe88 sp=0xc002ebfd38 pc=0x22c1472 +github.com/rancher/opni/pkg/gateway.(*SyncRequester).HandleAgentConnection(0xc0017e1740, {0x5b430b0, 0xc0027cce40}, {0x5b2bf70?, 0xc002e61a40}) + /home/ec2-user/opni/pkg/gateway/sync.go:54 +0x3a5 fp=0xc002ebff78 sp=0xc002ebfe88 pc=0x22c0325 +github.com/rancher/opni/pkg/gateway.(*multiConnectionHandler).HandleAgentConnection.func1() + /home/ec2-user/opni/pkg/gateway/grpc.go:40 +0x82 fp=0xc002ebffe0 sp=0xc002ebff78 pc=0x22b3a62 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ebffe8 sp=0xc002ebffe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.(*multiConnectionHandler).HandleAgentConnection in goroutine 1644 + /home/ec2-user/opni/pkg/gateway/grpc.go:40 +0x6f + +goroutine 1647 [chan receive]: +runtime.gopark(0x4aef8c?, 0x4b0a55?, 0xb0?, 0x40?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ee5cd8 sp=0xc002ee5cb8 pc=0x4777ae +runtime.chanrecv(0xc00235d3e0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002ee5d50 sp=0xc002ee5cd8 pc=0x441325 +runtime.chanrecv1(0xc0027cce40?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002ee5d78 sp=0xc002ee5d50 pc=0x440f72 +github.com/rancher/opni/pkg/gateway.(*DelegateServer).HandleAgentConnection(0xc000ee40a0, {0x5b430b0, 0xc0027cce40}, {0x5b2bf70, 0xc002e61a40}) + /home/ec2-user/opni/pkg/gateway/delegate.go:74 +0x639 fp=0xc002ee5f78 sp=0xc002ee5d78 pc=0x22a7cd9 +github.com/rancher/opni/pkg/gateway.(*multiConnectionHandler).HandleAgentConnection.func1() + /home/ec2-user/opni/pkg/gateway/grpc.go:40 +0x82 fp=0xc002ee5fe0 sp=0xc002ee5f78 pc=0x22b3a62 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ee5fe8 sp=0xc002ee5fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.(*multiConnectionHandler).HandleAgentConnection in goroutine 1644 + /home/ec2-user/opni/pkg/gateway/grpc.go:40 +0x6f + +goroutine 1650 [select]: +runtime.gopark(0xc002f85410?, 0x2?, 0x0?, 0x0?, 0xc002f85394?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002f85200 sp=0xc002f851e0 pc=0x4777ae +runtime.selectgo(0xc002f85410, 0xc002f85390, 0xc00097fb40?, 0x0, 0x44e7c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002f85350 sp=0xc002f85200 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc002c2f810, {0xc002c31d80, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc002f85440 sp=0xc002f85350 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002c2f810, {0xc002c31d80, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc002f854d8 sp=0xc002f85440 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002c31d40, {0xc002c31d80, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002f85520 sp=0xc002f854d8 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002c31d40}, {0xc002c31d80, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002f85568 sp=0xc002f85520 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d8b320, {0xc002c31d80, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002f855d8 sp=0xc002f85568 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002c31d70, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002f856a8 sp=0xc002f855d8 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc002f857b0?, 0xc001d8b320, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002f85798 sp=0xc002f856a8 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002c31d70, {0x7f424347ae80, 0x779c980}, 0x5b18680?, {0x0, 0x0}, {0x4592b00, 0xc002f8af60}, 0xc00?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002f85850 sp=0xc002f85798 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc002c29790, {0x4592b00?, 0xc002f8af60}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc002f85ab0 sp=0xc002f85850 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001d8b194?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc002f85b10 sp=0xc002f85ab0 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001d8b0e0, 0xc002f85cc8, 0xc002f85c98) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc002f85bb8 sp=0xc002f85b10 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8b0e0, {0x4592b00, 0xc002f8af60}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc002f85cf8 sp=0xc002f85bb8 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc002852140, {0x4592b00, 0xc002f8af60}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc002f85ec0 sp=0xc002f85cf8 pc=0x12ac5b9 +github.com/rancher/opni/pkg/management.NewServer.unknownServiceHandler.func3.1() + /home/ec2-user/opni/pkg/management/extensions.go:538 +0x8c fp=0xc002f85f38 sp=0xc002f85ec0 pc=0x231dbec +golang.org/x/sync/errgroup.(*Group).Go.func1() + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:75 +0x77 fp=0xc002f85fe0 sp=0xc002f85f38 pc=0x2007477 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002f85fe8 sp=0xc002f85fe0 pc=0x4ad421 +created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1486 + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:72 +0x125 + +goroutine 1698 [select]: +runtime.gopark(0xc002e9bf70?, 0x2?, 0xb4?, 0x1b?, 0xc002e9bebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e9bd20 sp=0xc002e9bd00 pc=0x4777ae +runtime.selectgo(0xc002e9bf70, 0xc002e9beb8, 0xc00215fe40?, 0x0, 0xc00215fef0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e9be70 sp=0xc002e9bd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002cc3be0, {0x5b426e0, 0xc002ef5f90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc002e9bfa0 sp=0xc002e9be70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc002e9bfe0 sp=0xc002e9bfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e9bfe8 sp=0xc002e9bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 609 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1714 [IO wait]: +runtime.gopark(0x440100?, 0xc00313f878?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00313f830 sp=0xc00313f810 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00313f868 sp=0xc00313f830 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c43d0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00313f888 sp=0xc00313f868 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc003058020, 0xc002ff6000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00313f8d0 sp=0xc00313f888 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc003058000, {0xc002ff6000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00313f9c0 sp=0xc00313f8d0 pc=0x50e5a5 +net.(*netFD).Read(0xc003058000, {0xc002ff6000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00313fa20 sp=0xc00313f9c0 pc=0x89342b +net.(*conn).Read(0xc002f9a458, {0xc002ff6000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00313faa8 sp=0xc00313fa20 pc=0x8b176d +net.(*TCPConn).Read(0xc002f9a458, {0xc002ff6000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc00313faf0 sp=0xc00313faa8 pc=0x8d72f4 +bufio.(*Reader).Read(0xc002fd8fc0, {0xc002686900, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00313fbb0 sp=0xc00313faf0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc002fd8fc0}, {0xc002686900, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00313fbf8 sp=0xc00313fbb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc002686900, 0x9, 0x9}, {0x5b0f0a0, 0xc002fd8fc0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00313fc98 sp=0xc00313fbf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0026868c0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00313fdd0 sp=0xc00313fc98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc002028240, 0xa9aef4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc00313ffb0 sp=0xc00313fdd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc00313ffe0 sp=0xc00313ffb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00313ffe8 sp=0xc00313ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1699 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 1649 [chan receive]: +runtime.gopark(0x4aef8c?, 0x4b0a55?, 0xf0?, 0x1c?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b59e58 sp=0xc001b59e38 pc=0x4777ae +runtime.chanrecv(0xc000c9a580, 0xc001b59f10, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b59ed0 sp=0xc001b59e58 pc=0x441325 +runtime.chanrecv2(0xc002731cf0?, 0x44642c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc001b59ef8 sp=0xc001b59ed0 pc=0x440f92 +github.com/nats-io/nats%2ego.(*js).resetPendingAcksOnReconnect(0xc002731ce0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/js.go:692 +0x9c fp=0xc001b59fc0 sp=0xc001b59ef8 pc=0x1f3cd9c +github.com/nats-io/nats%2ego.(*js).newAsyncReply.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/js.go:672 +0x34 fp=0xc001b59fe0 sp=0xc001b59fc0 pc=0x1f3ccd4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b59fe8 sp=0xc001b59fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*js).newAsyncReply in goroutine 1139 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/js.go:672 +0x88d + +goroutine 1548 [select]: +runtime.gopark(0xc002fb16a0?, 0x2?, 0x0?, 0x0?, 0xc002fb164c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002fb14b8 sp=0xc002fb1498 pc=0x4777ae +runtime.selectgo(0xc002fb16a0, 0xc002fb1648, 0x7f429a4abf00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002fb1608 sp=0xc002fb14b8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0027cafa0, {0xc0027cd540, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002fb16d0 sp=0xc002fb1608 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0027cafa0, {0xc0027cd540, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002fb1768 sp=0xc002fb16d0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0027cd4a0, {0xc0027cd540, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002fb17b0 sp=0xc002fb1768 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0027cd4a0}, {0xc0027cd540, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002fb17f8 sp=0xc002fb17b0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0028a0fc0, {0xc0027cd540, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002fb1868 sp=0xc002fb17f8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0027cd530, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002fb1938 sp=0xc002fb1868 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x0?, 0xc0028a0fc0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002fb1a28 sp=0xc002fb1938 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0027cd530, {0x7f424347ae80, 0x779c980}, 0xc001d8b194?, {0x0, 0x0}, {0x435c620, 0xc002e5dbc0}, 0x4b057b?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002fb1ae0 sp=0xc002fb1a28 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc00278a2d0, {0x435c620?, 0xc002e5dbc0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002fb1d18 sp=0xc002fb1ae0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0027cd680, {0x435c620, 0xc002e5dbc0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002fb1e00 sp=0xc002fb1d18 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002d9c200, {0x435c620, 0xc002e5dbc0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002fb1e38 sp=0xc002fb1e00 pc=0x1fd94b2 +github.com/rancher/opni/plugins/alerting/apis/alertops.(*configReconcilerSyncConfigServer).Recv(0xc002db02a0) + /home/ec2-user/opni/plugins/alerting/apis/alertops/alertops_grpc.pb.go:487 +0x66 fp=0xc002fb1e78 sp=0xc002fb1e38 pc=0x2610ca6 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).SyncConfig.func1() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/admin.go:282 +0x1b8 fp=0xc002fb1fa0 sp=0xc002fb1e78 pc=0x395c078 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc002fb1fe0 sp=0xc002fb1fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002fb1fe8 sp=0xc002fb1fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1547 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1651 [select]: +runtime.gopark(0xc002333738?, 0x2?, 0x0?, 0x0?, 0xc0023336e4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002333550 sp=0xc002333530 pc=0x4777ae +runtime.selectgo(0xc002333738, 0xc0023336e0, 0xc0023336e8?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0023336a0 sp=0xc002333550 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002c2f590, {0xc002c31900, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002333768 sp=0xc0023336a0 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002c2f590, {0xc002c31900, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002333800 sp=0xc002333768 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002c31860, {0xc002c31900, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002333848 sp=0xc002333800 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002c31860}, {0xc002c31900, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002333890 sp=0xc002333848 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d8afc0, {0xc002c31900, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002333900 sp=0xc002333890 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002c318f0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0023339d0 sp=0xc002333900 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc002f8aea0?, 0xc001d8afc0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002333ac0 sp=0xc0023339d0 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002c318f0, {0x7f424347ae80, 0x779c980}, 0xc002cd4ce8?, {0x0, 0x0}, {0x4592b00, 0xc002f8af90}, 0x96?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002333b78 sp=0xc002333ac0 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001670e10, {0x4592b00?, 0xc002f8af90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002333db0 sp=0xc002333b78 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002c31a40, {0x4592b00, 0xc002f8af90}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002333e98 sp=0xc002333db0 pc=0x12b01b3 +github.com/rancher/opni/pkg/management.NewServer.unknownServiceHandler.func3.2() + /home/ec2-user/opni/pkg/management/extensions.go:551 +0x9a fp=0xc002333f38 sp=0xc002333e98 pc=0x231da1a +golang.org/x/sync/errgroup.(*Group).Go.func1() + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:75 +0x77 fp=0xc002333fe0 sp=0xc002333f38 pc=0x2007477 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002333fe8 sp=0xc002333fe0 pc=0x4ad421 +created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1486 + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:72 +0x125 + +goroutine 1549 [select]: +runtime.gopark(0xc002c776a0?, 0x2?, 0x0?, 0x0?, 0xc002c7764c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c774b8 sp=0xc002c77498 pc=0x4777ae +runtime.selectgo(0xc002c776a0, 0xc002c77648, 0x7f429a4ac200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c77608 sp=0xc002c774b8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002c2fdb0, {0xc002c72730, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002c776d0 sp=0xc002c77608 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002c2fdb0, {0xc002c72730, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002c77768 sp=0xc002c776d0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002c72690, {0xc002c72730, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002c777b0 sp=0xc002c77768 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002c72690}, {0xc002c72730, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002c777f8 sp=0xc002c777b0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d8bb00, {0xc002c72730, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002c77868 sp=0xc002c777f8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002c72720, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002c77938 sp=0xc002c77868 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc001d8bb00, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002c77a28 sp=0xc002c77938 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002c72720, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x435c620, 0xc00309a4e0}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002c77ae0 sp=0xc002c77a28 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001670ff0, {0x435c620?, 0xc00309a4e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002c77d18 sp=0xc002c77ae0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002c72870, {0x435c620, 0xc00309a4e0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002c77e00 sp=0xc002c77d18 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002bf92c0, {0x435c620, 0xc00309a4e0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002c77e38 sp=0xc002c77e00 pc=0x1fd94b2 +github.com/rancher/opni/plugins/alerting/apis/alertops.(*configReconcilerSyncConfigServer).Recv(0xc002c7e0c0) + /home/ec2-user/opni/plugins/alerting/apis/alertops/alertops_grpc.pb.go:487 +0x66 fp=0xc002c77e78 sp=0xc002c77e38 pc=0x2610ca6 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).SyncConfig.func1() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/admin.go:282 +0x1b8 fp=0xc002c77fa0 sp=0xc002c77e78 pc=0x395c078 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc002c77fe0 sp=0xc002c77fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c77fe8 sp=0xc002c77fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1574 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1562 [select]: +runtime.gopark(0xc002d7f410?, 0x2?, 0x0?, 0x0?, 0xc002d7f394?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d7f200 sp=0xc002d7f1e0 pc=0x4777ae +runtime.selectgo(0xc002d7f410, 0xc002d7f390, 0xc00097fb40?, 0x0, 0x44e7c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d7f350 sp=0xc002d7f200 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc002c2fc70, {0xc002c724f0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc002d7f440 sp=0xc002d7f350 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002c2fc70, {0xc002c724f0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc002d7f4d8 sp=0xc002d7f440 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002c724b0, {0xc002c724f0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002d7f520 sp=0xc002d7f4d8 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002c724b0}, {0xc002c724f0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002d7f568 sp=0xc002d7f520 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d8b9e0, {0xc002c724f0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002d7f5d8 sp=0xc002d7f568 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002c724e0, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002d7f6a8 sp=0xc002d7f5d8 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc002d7f7b0?, 0xc001d8b9e0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002d7f798 sp=0xc002d7f6a8 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002c724e0, {0x7f424347ae80, 0x779c980}, 0x5b18680?, {0x0, 0x0}, {0x4592b00, 0xc002d1def0}, 0xc00?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002d7f850 sp=0xc002d7f798 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc002c29860, {0x4592b00?, 0xc002d1def0}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc002d7fab0 sp=0xc002d7f850 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001d8b614?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc002d7fb10 sp=0xc002d7fab0 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001d8b560, 0xc002d7fcc8, 0xc002d7fc98) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc002d7fbb8 sp=0xc002d7fb10 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8b560, {0x4592b00, 0xc002d1def0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc002d7fcf8 sp=0xc002d7fbb8 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc002852400, {0x4592b00, 0xc002d1def0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc002d7fec0 sp=0xc002d7fcf8 pc=0x12ac5b9 +github.com/rancher/opni/pkg/management.NewServer.unknownServiceHandler.func3.1() + /home/ec2-user/opni/pkg/management/extensions.go:538 +0x8c fp=0xc002d7ff38 sp=0xc002d7fec0 pc=0x231dbec +golang.org/x/sync/errgroup.(*Group).Go.func1() + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:75 +0x77 fp=0xc002d7ffe0 sp=0xc002d7ff38 pc=0x2007477 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d7ffe8 sp=0xc002d7ffe0 pc=0x4ad421 +created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1570 + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:72 +0x125 + +goroutine 1563 [select]: +runtime.gopark(0xc002d61738?, 0x2?, 0x20?, 0xdf?, 0xc002d616e4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d61550 sp=0xc002d61530 pc=0x4777ae +runtime.selectgo(0xc002d61738, 0xc002d616e0, 0x7f429a4ab000?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d616a0 sp=0xc002d61550 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002c2f9f0, {0xc002c72070, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002d61768 sp=0xc002d616a0 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002c2f9f0, {0xc002c72070, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002d61800 sp=0xc002d61768 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002c31fb0, {0xc002c72070, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002d61848 sp=0xc002d61800 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002c31fb0}, {0xc002c72070, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002d61890 sp=0xc002d61848 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d8b440, {0xc002c72070, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002d61900 sp=0xc002d61890 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002c72060, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002d619d0 sp=0xc002d61900 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x60?, 0xc001d8b440, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002d61ac0 sp=0xc002d619d0 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002c72060, {0x7f424347ae80, 0x779c980}, 0xc002c2fae0?, {0x0, 0x0}, {0x4592b00, 0xc002f8b860}, 0xc002c723f0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002d61b78 sp=0xc002d61ac0 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001670f00, {0x4592b00?, 0xc002f8b860}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002d61db0 sp=0xc002d61b78 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002c721b0, {0x4592b00, 0xc002f8b860}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002d61e98 sp=0xc002d61db0 pc=0x12b01b3 +github.com/rancher/opni/pkg/management.NewServer.unknownServiceHandler.func3.2() + /home/ec2-user/opni/pkg/management/extensions.go:551 +0x9a fp=0xc002d61f38 sp=0xc002d61e98 pc=0x231da1a +golang.org/x/sync/errgroup.(*Group).Go.func1() + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:75 +0x77 fp=0xc002d61fe0 sp=0xc002d61f38 pc=0x2007477 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d61fe8 sp=0xc002d61fe0 pc=0x4ad421 +created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1570 + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:72 +0x125 + +goroutine 1564 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x5e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017a5e20 sp=0xc0017a5e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0017a5e98 sp=0xc0017a5e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x1102359?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0017a5ec0 sp=0xc0017a5e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc0017a5fe0 sp=0xc0017a5ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017a5fe8 sp=0xc0017a5fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1565 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x4e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cd4e20 sp=0xc002cd4e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cd4e98 sp=0xc002cd4e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0xc000f3e6e8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cd4ec0 sp=0xc002cd4e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cd4fe0 sp=0xc002cd4ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cd4fe8 sp=0xc002cd4fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1566 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x6e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c66e20 sp=0xc002c66e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c66e98 sp=0xc002c66e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x6a55575255575268?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c66ec0 sp=0xc002c66e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c66fe0 sp=0xc002c66ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c66fe8 sp=0xc002c66fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1567 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x7e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c67e20 sp=0xc002c67e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c67e98 sp=0xc002c67e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c67ec0 sp=0xc002c67e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c67fe0 sp=0xc002c67ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c67fe8 sp=0xc002c67fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1568 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x8e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c68e20 sp=0xc002c68e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c68e98 sp=0xc002c68e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c68ec0 sp=0xc002c68e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c68fe0 sp=0xc002c68ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c68fe8 sp=0xc002c68fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1569 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0xe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e70e20 sp=0xc002e70e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e70e98 sp=0xc002e70e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0xc002009ee0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e70ec0 sp=0xc002e70e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e70fe0 sp=0xc002e70ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e70fe8 sp=0xc002e70fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1666 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x2e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e72e20 sp=0xc002e72e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e72e98 sp=0xc002e72e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x2106cfa?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e72ec0 sp=0xc002e72e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e72fe0 sp=0xc002e72ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e72fe8 sp=0xc002e72fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1667 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x1e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e71e20 sp=0xc002e71e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e71e98 sp=0xc002e71e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x2166b62?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e71ec0 sp=0xc002e71e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e71fe0 sp=0xc002e71ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e71fe8 sp=0xc002e71fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1668 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0xbe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e7be20 sp=0xc002e7be00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e7be98 sp=0xc002e7be20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e7bec0 sp=0xc002e7be98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e7bfe0 sp=0xc002e7bec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e7bfe8 sp=0xc002e7bfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1669 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0xce?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e7ce20 sp=0xc002e7ce00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e7ce98 sp=0xc002e7ce20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x448845?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e7cec0 sp=0xc002e7ce98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e7cfe0 sp=0xc002e7cec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e7cfe8 sp=0xc002e7cfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1670 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0xde?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e7de20 sp=0xc002e7de00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e7de98 sp=0xc002e7de20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x2106cfa?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e7dec0 sp=0xc002e7de98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e7dfe0 sp=0xc002e7dec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e7dfe8 sp=0xc002e7dfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1671 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x5e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002fc5e20 sp=0xc002fc5e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002fc5e98 sp=0xc002fc5e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x5b50438?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002fc5ec0 sp=0xc002fc5e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002fc5fe0 sp=0xc002fc5ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002fc5fe8 sp=0xc002fc5fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1672 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0xee?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e7ee20 sp=0xc002e7ee00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e7ee98 sp=0xc002e7ee20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x1102359?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e7eec0 sp=0xc002e7ee98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e7efe0 sp=0xc002e7eec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e7efe8 sp=0xc002e7efe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1673 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x8e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e78e20 sp=0xc002e78e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e78e98 sp=0xc002e78e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0xc000f3e6e8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e78ec0 sp=0xc002e78e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e78fe0 sp=0xc002e78ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e78fe8 sp=0xc002e78fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1674 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0xae?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e7ae20 sp=0xc002e7ae00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e7ae98 sp=0xc002e7ae20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e7aec0 sp=0xc002e7ae98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e7afe0 sp=0xc002e7aec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e7afe8 sp=0xc002e7afe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1675 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x9e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e79e20 sp=0xc002e79e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e79e98 sp=0xc002e79e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0xc001150090?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e79ec0 sp=0xc002e79e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e79fe0 sp=0xc002e79ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e79fe8 sp=0xc002e79fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1676 [chan receive]: +runtime.gopark(0xc0024fc698?, 0x21c0d85?, 0x50?, 0xc7?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002fc1ef0 sp=0xc002fc1ed0 pc=0x4777ae +runtime.chanrecv(0xc002e0a960, 0xc002fc1fc0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002fc1f68 sp=0xc002fc1ef0 pc=0x441325 +runtime.chanrecv1(0x0?, 0xc0024fc7d0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002fc1f90 sp=0xc002fc1f68 pc=0x440f72 +github.com/rancher/opni/pkg/util/future.NewFromChannel[...].func1() + /home/ec2-user/opni/pkg/util/future/future.go:33 +0x48 fp=0xc002fc1fe0 sp=0xc002fc1f90 pc=0x20128e8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002fc1fe8 sp=0xc002fc1fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/util/future.NewFromChannel[...] in goroutine 1478 + /home/ec2-user/opni/pkg/util/future/future.go:32 +0x1ea + +goroutine 1656 [select]: +runtime.gopark(0xc001549f40?, 0x2?, 0xb4?, 0x1b?, 0xc001549ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001549d48 sp=0xc001549d28 pc=0x4777ae +runtime.selectgo(0xc001549f40, 0xc001549ec8, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001549e98 sp=0xc001549d48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002847680}, 0xc002949890, 0xc002fc7cb0, 0xc0022ee1e0, 0xc002fb9550) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc001549f70 sp=0xc001549e98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc001549fe0 sp=0xc001549f70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001549fe8 sp=0xc001549fe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1505 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1686 [select]: +runtime.gopark(0xc0030f7e48?, 0x2?, 0x0?, 0x0?, 0xc0030f7dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0030f7c30 sp=0xc0030f7c10 pc=0x4777ae +runtime.selectgo(0xc0030f7e48, 0xc0030f7dc8, 0xc0030f7dd8?, 0x0, 0xc001f4ec88?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0030f7d80 sp=0xc0030f7c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0030b8e60, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0030f7e78 sp=0xc0030f7d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0030d2150) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0030f7f30 sp=0xc0030f7e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc0030f7fe0 sp=0xc0030f7f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0030f7fe8 sp=0xc0030f7fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1685 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1627 [select]: +runtime.gopark(0xc00300ff40?, 0x2?, 0xb4?, 0x1b?, 0xc00300fecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00300fd48 sp=0xc00300fd28 pc=0x4777ae +runtime.selectgo(0xc00300ff40, 0xc00300fec8, 0x4ad421?, 0x0, 0x1381849?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00300fe98 sp=0xc00300fd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc0027ca5f0}, 0xc0004f1b90, 0xc002efb500, 0xc002047680, 0xc002ef2ed0) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc00300ff70 sp=0xc00300fe98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc00300ffe0 sp=0xc00300ff70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00300ffe8 sp=0xc00300ffe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1619 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1683 [select]: +runtime.gopark(0xc0011fff40?, 0x2?, 0xb4?, 0x1b?, 0xc0011ffecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011ffd48 sp=0xc0011ffd28 pc=0x4777ae +runtime.selectgo(0xc0011fff40, 0xc0011ffec8, 0xc0030a9ed0?, 0x0, 0xc0030ab1c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011ffe98 sp=0xc0011ffd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002cba0f0}, 0xc002949ad0, 0xc0030ab560, 0xc0022ef620, 0xc002f7f570) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc0011fff70 sp=0xc0011ffe98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc0011fffe0 sp=0xc0011fff70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011fffe8 sp=0xc0011fffe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1536 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1715 [select]: +runtime.gopark(0xc003135e70?, 0x2?, 0x0?, 0x0?, 0xc003135df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc003135c58 sp=0xc003135c38 pc=0x4777ae +runtime.selectgo(0xc003135e70, 0xc003135df0, 0xc002fe08c0?, 0x0, 0xc002f9a458?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc003135da8 sp=0xc003135c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002fea730, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc003135ea0 sp=0xc003135da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002fe08c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc003135f58 sp=0xc003135ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc003135fe0 sp=0xc003135f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc003135fe8 sp=0xc003135fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1699 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1633 [select]: +runtime.gopark(0xc002fadf70?, 0x2?, 0xb4?, 0x1b?, 0xc002fadebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002fadd20 sp=0xc002fadd00 pc=0x4777ae +runtime.selectgo(0xc002fadf70, 0xc002fadeb8, 0xc002000000?, 0x0, 0xa9fd34?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002fade70 sp=0xc002fadd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002cc3b80, {0x5b426e0, 0xc002ef5f40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc002fadfa0 sp=0xc002fade70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc002fadfe0 sp=0xc002fadfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002fadfe8 sp=0xc002fadfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 609 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1687 [select]: +runtime.gopark(0xc002fbef70?, 0x4?, 0x28?, 0x8e?, 0xc002fbedd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002fbec00 sp=0xc002fbebe0 pc=0x4777ae +runtime.selectgo(0xc002fbef70, 0xc002fbedd0, 0x40a16a0?, 0x0, 0xc002f7fbe0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002fbed50 sp=0xc002fbec00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002f2e4e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc002fbefc0 sp=0xc002fbed50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc002fbefe0 sp=0xc002fbefc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002fbefe8 sp=0xc002fbefe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1685 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1688 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002faf7d8 sp=0xc002faf7b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002faf810 sp=0xc002faf7d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c42d8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002faf830 sp=0xc002faf810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc002da12a0, 0xc0030d6000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002faf878 sp=0xc002faf830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc002da1280, {0xc0030d6000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc002faf968 sp=0xc002faf878 pc=0x50e5a5 +net.(*netFD).Read(0xc002da1280, {0xc0030d6000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc002faf9c8 sp=0xc002faf968 pc=0x89342b +net.(*conn).Read(0xc001f4ec88, {0xc0030d6000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc002fafa50 sp=0xc002faf9c8 pc=0x8b176d +net.(*TCPConn).Read(0xc001f4ec88, {0xc0030d6000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc002fafa98 sp=0xc002fafa50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc00309bc20, {0xc002f4f620, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc002fafb58 sp=0xc002fafa98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00309bc20}, {0xc002f4f620, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002fafba0 sp=0xc002fafb58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc002f4f620, 0x9, 0x9}, {0x5b0f0a0, 0xc00309bc20}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc002fafc40 sp=0xc002fafba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc002f4f5e0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc002fafd78 sp=0xc002fafc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002f2e4e0, 0x4ad421?, 0xc000f3f7d0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc002fafec8 sp=0xc002fafd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c1e0, {0x5b54e00?, 0xc002f2e4e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc002faff80 sp=0xc002fafec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc002faffe0 sp=0xc002faff80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002faffe8 sp=0xc002faffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1685 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +Ginkgo ran 1 suite in 9.808135734s + +Test Suite Failed diff --git a/test/plugins/alerting/slo_test.go b/test/plugins/alerting/slo_test.go new file mode 100644 index 0000000000..756ffee1ee --- /dev/null +++ b/test/plugins/alerting/slo_test.go @@ -0,0 +1,564 @@ +package alerting_test + +import ( + "context" + "fmt" + "time" + + "github.com/golang/mock/gomock" + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "github.com/rancher/opni/pkg/apis/core/v1" + managementv1 "github.com/rancher/opni/pkg/apis/management/v1" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/rancher/opni/pkg/capabilities" + "github.com/rancher/opni/pkg/capabilities/wellknown" + "github.com/rancher/opni/pkg/slo/backend" + "github.com/rancher/opni/pkg/test" + mockslo "github.com/rancher/opni/pkg/test/mock/slo" + "github.com/rancher/opni/pkg/test/testutil" + "github.com/rancher/opni/pkg/util" + "github.com/rancher/opni/plugins/alerting/apis/alertops" + "github.com/rancher/opni/plugins/alerting/pkg/alerting/slo/v1" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/emptypb" +) + +const ( + mockSource = "slo" + errorSource = "error" +) + +var _ = Describe("SLO Alerting", Ordered, Label("integration"), func() { + var sloClient slov1.SLOClient + BeforeAll(func() { + env = &test.Environment{} + Expect(env).NotTo(BeNil()) + Expect(env.Start()).To(Succeed()) + DeferCleanup(env.Stop) + + ctrl := gomock.NewController(GinkgoT()) + + errStore := mockslo.NewMockSLOStore(ctrl) + + errDiscovery := mockslo.NewMockServiceBackend(ctrl) + + errStore.EXPECT().Create(gomock.Any(), gomock.Any()).Return( + nil, + fmt.Errorf("some error"), + ).AnyTimes() + + errStore.EXPECT().Delete(gomock.Any(), gomock.Any()).Return( + fmt.Errorf("some error"), + ).AnyTimes() + + errStore.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return( + nil, + fmt.Errorf("some error"), + ).AnyTimes() + + errStore.EXPECT().Clone(gomock.Any(), gomock.Any()).Return( + nil, + nil, + fmt.Errorf("some error"), + ).AnyTimes() + + errStore.EXPECT().MultiClusterClone(gomock.Any(), gomock.Any(), gomock.Any()).Return( + nil, + nil, + []error{fmt.Errorf("some error")}, + ).AnyTimes() + + errStore.EXPECT().Status(gomock.Any(), gomock.Any()).Return( + nil, + fmt.Errorf("some error"), + ).AnyTimes() + + errStore.EXPECT().Preview(gomock.Any(), gomock.Any()).Return( + nil, + fmt.Errorf("some error"), + ).AnyTimes() + + errDiscovery.EXPECT().ListEvents(gomock.Any(), gomock.Any()).Return( + nil, + fmt.Errorf("some error"), + ).AnyTimes() + + errDiscovery.EXPECT().ListMetrics(gomock.Any(), gomock.Any()).Return( + nil, + fmt.Errorf("some error"), + ).AnyTimes() + + errDiscovery.EXPECT().ListServices(gomock.Any(), gomock.Any()).Return( + nil, + fmt.Errorf("some error"), + ).AnyTimes() + + sloStore := mockslo.NewMockSLOStore(ctrl) + sloDiscovery := mockslo.NewMockServiceBackend(ctrl) + + sloDiscovery.EXPECT().ListServices(gomock.Any(), gomock.Any()).Return( + &slov1.ServiceList{ + Items: []*slov1.Service{ + { + ServiceId: "test-service", + ClusterId: "some-cluster", + }, + }, + }, + nil, + ).AnyTimes() + + sloDiscovery.EXPECT().ListMetrics(gomock.Any(), gomock.Any()).Return( + &slov1.MetricGroupList{ + GroupNameToMetrics: map[string]*slov1.MetricList{ + "testgroup": { + Items: []*slov1.Metric{ + { + Id: "testmetric", + }, + }, + }, + }, + }, + nil, + ).AnyTimes() + + sloDiscovery.EXPECT().ListEvents(gomock.Any(), gomock.Any()).Return( + &slov1.EventList{ + Items: []*slov1.Event{ + { + Key: "test-event", + Vals: []string{"a", "b", "c"}, + }, + }, + }, + nil, + ).AnyTimes() + + sloStore.EXPECT().Create(gomock.Any(), gomock.Any()).Return( + &corev1.Reference{ + Id: "test-slo", + }, + nil, + ).AnyTimes() + + sloStore.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return( + &slov1.SLOData{ + Id: "test-slo", + SLO: util.ProtoClone(updatedMockSLO), + }, + nil, + ).AnyTimes() + + sloStore.EXPECT().Delete(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + + sloStore.EXPECT().Clone(gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, toClone *slov1.SLOData) (*corev1.Reference, *slov1.SLOData, error) { + id := uuid.New().String() + clonedData := util.ProtoClone(toClone) + clonedData.Id = id + return &corev1.Reference{ + Id: id, + }, + clonedData, + nil + }).AnyTimes() + + sloStore.EXPECT().MultiClusterClone(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func( + _ context.Context, + slo *slov1.SLOData, + toClusters []*corev1.Reference, + ) ([]*corev1.Reference, []*slov1.SLOData, []error) { + ids := []*corev1.Reference{} + datas := []*slov1.SLOData{} + errors := []error{} + for _, cluster := range toClusters { + uuid := uuid.New().String() + newSLO := util.ProtoClone( + slo.GetSLO(), + ) + newSLO.ClusterId = cluster.Id + newSLO.Name += "-clone" + ids = append(ids, &corev1.Reference{Id: uuid}) + datas = append(datas, &slov1.SLOData{ + Id: uuid, + SLO: newSLO, + }) + errors = append(errors, nil) + } + return ids, datas, errors + }).AnyTimes() + + sloStore.EXPECT().Status(gomock.Any(), gomock.Any()).Return( + &slov1.SLOStatus{ + State: slov1.SLOStatusState_Creating, + }, + nil, + ).AnyTimes() + + sloStore.EXPECT().Preview(gomock.Any(), gomock.Any()).Return( + &slov1.SLOPreviewResponse{ + PlotVector: &slov1.PlotVector{ + Objective: 99.9, + Items: []*slov1.DataPoint{}, + Windows: []*slov1.AlertFiringWindows{}, + }, + }, + nil, + ).AnyTimes() + + alertClusterClient := alertops.NewAlertingAdminClient(env.ManagementClientConn()) + mgmtClient := env.NewManagementClient() + + mockDatasource := backend.NewSLODatasource(sloStore, sloDiscovery, func(_ context.Context, _ *corev1.Reference) error { + return nil + }) + + errDatasource := backend.NewSLODatasource(errStore, errDiscovery, func(_ context.Context, _ *corev1.Reference) error { + return nil + }) + + slo.RegisterDatasource(mockSource, mockDatasource) + slo.RegisterDatasource(errorSource, errDatasource) + + _, err := alertClusterClient.InstallCluster(env.Context(), &emptypb.Empty{}) + Expect(err).To(BeNil()) + + certsInfo, err := mgmtClient.CertsInfo(context.Background(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + fingerprint := certsInfo.Chain[len(certsInfo.Chain)-1].Fingerprint + Expect(fingerprint).NotTo(BeEmpty()) + token, err := mgmtClient.CreateBootstrapToken(context.Background(), &managementv1.CreateBootstrapTokenRequest{ + Ttl: durationpb.New(1 * time.Hour), + }) + Expect(err).NotTo(HaveOccurred()) + _, errC := env.StartAgent("agent", token, []string{fingerprint}) + Eventually(errC, time.Second*10, time.Millisecond*200).Should(Receive(BeNil())) + + _, errC2 := env.StartAgent("agent2", token, []string{fingerprint}) + Eventually(errC2, time.Second*10, time.Millisecond*200).Should(Receive(BeNil())) + + Eventually(func() error { + status, err := alertClusterClient.GetClusterStatus(env.Context(), &emptypb.Empty{}) + if err != nil { + return err + } + if status.State != alertops.InstallState_Installed { + return fmt.Errorf("alerting cluster install state is %s", status.State.String()) + } + return nil + }, time.Second*5, time.Millisecond*200).Should(Succeed()) + + sloClient = slov1.NewSLOClient(env.ManagementClientConn()) + }) + + When("interacting with the SLO server", func() { + It("should list the available datasources", func() { + datasources, err := sloClient.ListDatasources(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + Expect(datasources.Items).To(ConsistOf([]string{errorSource, mockSource, "monitoring"})) + }) + }) + + When("Managing SLOs with the mocked datasource", func() { + It("should return a list of available services", func() { + svcList, err := sloClient.ListServices(env.Context(), &slov1.ListServicesRequest{ + Datasource: mockSource, + ClusterId: "some-cluster", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(svcList.Items).To(HaveLen(1)) + Expect(svcList.Items[0]).To( + Equal(&slov1.Service{ + ServiceId: "test-service", + ClusterId: "some-cluster", + }), + ) + }) + + It("should return a list of available metrics", func() { + metricList, err := sloClient.ListMetrics(env.Context(), &slov1.ListMetricsRequest{ + Datasource: mockSource, + ClusterId: "some-cluster", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(metricList.GroupNameToMetrics).To(HaveLen(1)) + Expect(metricList.GroupNameToMetrics["testgroup"].Items).To( + Equal( + []*slov1.Metric{ + { + Id: "testmetric", + }, + }, + ), + ) + }) + + It("should return a list of available events", func() { + eventList, err := sloClient.ListEvents(env.Context(), &slov1.ListEventsRequest{ + Datasource: mockSource, + ClusterId: "some-cluster", + ServiceId: "test-service", + MetricId: "testmetric", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(eventList.Items).To(HaveLen(1)) + Expect(eventList.Items[0]).To( + Equal(&slov1.Event{ + Key: "test-event", + Vals: []string{"a", "b", "c"}, + }), + ) + }) + + It("should create an SLO", func() { + sloImpl := util.ProtoClone(mockSLO) + sloRef, err := sloClient.CreateSLO(env.Context(), &slov1.CreateSLORequest{ + Slo: sloImpl, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(sloRef).NotTo(BeNil()) + + By("verifying we can fetch the SLO") + data, err := sloClient.GetSLO(env.Context(), sloRef) + Expect(err).NotTo(HaveOccurred()) + + Expect(data.SLO).To(testutil.ProtoEqual((sloImpl))) + + By("verifying the SLO list only contains one element") + sloList, err := sloClient.ListSLOs(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + Expect(sloList.Items).To(HaveLen(1)) + }) + + Specify("when the SLOStore fails to create an SLO, the management server should not have it", func() { + sloList, err := sloClient.ListSLOs(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + originalLen := len(sloList.Items) + + sloImpl := util.ProtoClone(mockSLO) + sloImpl.Datasource = errorSource + createSLOFailure := &slov1.CreateSLORequest{ + Slo: sloImpl, + } + + sloRef, err := sloClient.CreateSLO(env.Context(), createSLOFailure) + Expect(err).To(HaveOccurred()) + Expect(sloRef).To(BeNil()) + + sloList, err = sloClient.ListSLOs(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + Expect(sloList.Items).To(HaveLen(originalLen)) + }) + + It("should update the SLO details", func() { + sloImpl := util.ProtoClone(mockSLO) + _, err := sloClient.UpdateSLO(env.Context(), &slov1.SLOData{ + Id: "test-slo", + SLO: sloImpl, + }) + Expect(err).NotTo(HaveOccurred()) + + data, err := sloClient.GetSLO(env.Context(), &corev1.Reference{ + Id: "test-slo", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(data.SLO).To(testutil.ProtoEqual(updatedMockSLO)) + }) + + It("should clone an SLO", func() { + sloList, err := sloClient.ListSLOs(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + Expect(sloList.Items).NotTo(HaveLen(0)) + + toClone := sloList.GetItems()[0] + + clone, err := sloClient.CloneSLO(env.Context(), &corev1.Reference{Id: toClone.Id}) + Expect(err).NotTo(HaveOccurred()) + Expect(clone.SLO).To(testutil.ProtoEqual(toClone.GetSLO())) + }) + + It("should clone an SLO across clusters", func() { + sloList, err := sloClient.ListSLOs(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + Expect(sloList.Items).NotTo(HaveLen(0)) + + originalLen := len(sloList.Items) + + toClone := sloList.GetItems()[0] + + failures, err := sloClient.CloneToClusters(env.Context(), &slov1.MultiClusterSLO{ + CloneId: &corev1.Reference{Id: toClone.GetId()}, + Clusters: []*corev1.Reference{ + {Id: "agent"}, + {Id: "agent2"}, + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(failures.GetFailures()).To(HaveLen(0)) + + sloList, err = sloClient.ListSLOs(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + + Expect(sloList.Items).To(HaveLen(originalLen + 2)) + + }) + + It("should error when the fetching the status of non-existent SLO", func() { + status, err := sloClient.GetSLOStatus(env.Context(), &corev1.Reference{ + Id: "definitely does not exists", + }) + + Expect(err).To(HaveOccurred()) + Expect(status).To(BeNil()) + }) + + It("should report the status of running SLOs", func() { + slos, err := sloClient.ListSLOs(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + Expect(slos.GetItems()).NotTo(HaveLen(0)) + + for _, slo := range slos.GetItems() { + status, err := sloClient.GetSLOStatus(env.Context(), &corev1.Reference{ + Id: slo.GetId(), + }) + Expect(err).NotTo(HaveOccurred()) + Expect(status).NotTo(BeNil()) + Expect(status.State).To(Equal(slov1.SLOStatusState_Creating)) + } + }) + + It("should return a preview of an SLO definition", func() { + payload := util.ProtoClone(mockSLO) + vector, err := sloClient.Preview(env.Context(), &slov1.CreateSLORequest{ + Slo: payload, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(vector.GetPlotVector()).NotTo(BeNil()) + }) + + It("should delete SLOs", func() { + sloList, err := sloClient.ListSLOs(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + Expect(len(sloList.Items)).To(BeNumerically(">", 0)) + + for _, item := range sloList.GetItems() { + _, err = sloClient.DeleteSLO(env.Context(), &corev1.Reference{ + Id: item.GetId(), + }) + Expect(err).NotTo(HaveOccurred()) + } + + sloList, err = sloClient.ListSLOs(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + Expect(sloList.Items).To(HaveLen(0)) + }) + }) + + When("Managing SLOs with the metrics datasource", func() { + It("Should return failed precondition when metrics is not installed", func() { + mgmtClient := env.NewManagementClient() + cluster, err := mgmtClient.GetCluster(env.Context(), &corev1.Reference{Id: "agent"}) + Expect(err).NotTo(HaveOccurred()) + Expect(capabilities.Has(cluster, capabilities.Cluster(wellknown.CapabilityMetrics))).To(BeFalse()) + + _, err = sloClient.CreateSLO(env.Context(), &slov1.CreateSLORequest{ + Slo: metricsSLO, + }) + Expect(err).To(HaveOccurred()) + Expect(err).To(testutil.MatchStatusCode(codes.FailedPrecondition)) + + _, err = sloClient.CreateSLO(env.Context(), &slov1.CreateSLORequest{ + Slo: util.ProtoClone(metricsSLO), + }) + Expect(err).To(HaveOccurred()) + Expect(err).To(testutil.MatchStatusCode(codes.FailedPrecondition)) + + _, err = sloClient.ListServices(env.Context(), &slov1.ListServicesRequest{ + Datasource: "monitoring", + ClusterId: "agent", + }) + Expect(err).To(HaveOccurred()) + Expect(err).To(testutil.MatchStatusCode(codes.FailedPrecondition)) + + _, err = sloClient.ListEvents(env.Context(), &slov1.ListEventsRequest{ + Datasource: "monitoring", + ClusterId: "agent", + ServiceId: "test-service", + MetricId: "testmetric", + }) + Expect(err).To(HaveOccurred()) + + _, err = sloClient.ListMetrics(env.Context(), &slov1.ListMetricsRequest{ + Datasource: "monitoring", + ClusterId: "agent", + ServiceId: "test-service", + }) + Expect(err).To(HaveOccurred()) + Expect(err).To(testutil.MatchStatusCode(codes.FailedPrecondition)) + }) + }) +}) + +var ( + mockSLO = &slov1.ServiceLevelObjective{ + Name: "test-slo-success", + ClusterId: "agent", + ServiceId: "test-service", + GoodMetricName: "testmetric", + GoodEvents: []*slov1.Event{ + { + Key: "a", + Vals: []string{"a", "b", "c"}, + }, + }, + TotalMetricName: "testmetric", + TotalEvents: []*slov1.Event{}, + Target: &slov1.Target{Value: 99.9}, + SloPeriod: "30d", + BudgetingInterval: durationpb.New(time.Minute * 5), + Datasource: mockSource, + } + + metricsSLO = &slov1.ServiceLevelObjective{ + Name: "test-slo-success", + ClusterId: "agent", + ServiceId: "test-service", + GoodMetricName: "testmetric", + GoodEvents: []*slov1.Event{ + { + Key: "a", + Vals: []string{"a", "b", "c"}, + }, + }, + TotalMetricName: "testmetric", + TotalEvents: []*slov1.Event{}, + Target: &slov1.Target{Value: 99.9}, + SloPeriod: "30d", + BudgetingInterval: durationpb.New(time.Minute * 5), + Datasource: "monitoring", + } + + updatedMockSLO = &slov1.ServiceLevelObjective{ + Name: "test-slo-success2", + ClusterId: "agent", + ServiceId: "test-service2", + GoodMetricName: "testmetric2", + GoodEvents: []*slov1.Event{ + { + Key: "a2", + Vals: []string{"a2", "b2", "c2"}, + }, + }, + TotalMetricName: "testmetric2", + TotalEvents: []*slov1.Event{}, + Target: &slov1.Target{Value: 99.9}, + SloPeriod: "30d", + BudgetingInterval: durationpb.New(time.Minute * 5), + Datasource: mockSource, + } +) diff --git a/test/plugins/alerting/testlog.txt b/test/plugins/alerting/testlog.txt new file mode 100644 index 0000000000..ef7f96732c --- /dev/null +++ b/test/plugins/alerting/testlog.txt @@ -0,0 +1,5437 @@ +Running Suite: Alerting Suite - /home/ec2-user/opni/test/plugins/alerting +========================================================================= +Random Seed: 1692063585 + +Will run 40 of 41 specs +------------------------------ +[BeforeSuite]  +/home/ec2-user/opni/test/plugins/alerting/alerting_suite_test.go:46 + 2023-08-15T01:39:55Z INFO test.env Starting test environment + 2023-08-15T01:39:55Z INFO test.env Copying 7 files from embedded testdata/cortex to /tmp/opni-test-2132242568/cortex + 2023-08-15T01:39:55Z INFO test.env Waiting for jetstream to start... + [231479] 2023/08/15 01:39:55.563722 [INF] Starting nats-server + [231479] 2023/08/15 01:39:55.563769 [INF] Version: 2.9.21 + [231479] 2023/08/15 01:39:55.563772 [INF] Git: [b2e7725] + [231479] 2023/08/15 01:39:55.563798 [INF] Name: NDB2A7RMGWYYYSG3TTANYY53LKD2LU7AGFTQ7AIYHAYGKNAHXFE3EGX7 + [231479] 2023/08/15 01:39:55.563806 [INF] Node: B5UBIPgv + [231479] 2023/08/15 01:39:55.563808 [INF] ID: NDB2A7RMGWYYYSG3TTANYY53LKD2LU7AGFTQ7AIYHAYGKNAHXFE3EGX7 + [231479] 2023/08/15 01:39:55.563817 [INF] Using configuration file: /tmp/opni-test-2132242568/jetstream/jetstream.conf + [231479] 2023/08/15 01:39:55.564279 [INF] Starting JetStream + [231479] 2023/08/15 01:39:55.564540 [INF] _ ___ _____ ___ _____ ___ ___ _ __ __ + [231479] 2023/08/15 01:39:55.564554 [INF] _ | | __|_ _/ __|_ _| _ \ __| /_\ | \/ | + [231479] 2023/08/15 01:39:55.564557 [INF] | || | _| | | \__ \ | | | / _| / _ \| |\/| | + [231479] 2023/08/15 01:39:55.564560 [INF] \__/|___| |_| |___/ |_| |_|_\___/_/ \_\_| |_| + [231479] 2023/08/15 01:39:55.564563 [INF] + [231479] 2023/08/15 01:39:55.564566 [INF] https://docs.nats.io/jetstream + [231479] 2023/08/15 01:39:55.564569 [INF] + [231479] 2023/08/15 01:39:55.564572 [INF] ---------------- JETSTREAM ---------------- + [231479] 2023/08/15 01:39:55.564578 [INF] Max Memory: 11.71 GB + [231479] 2023/08/15 01:39:55.564582 [INF] Max Storage: 5.29 GB + [231479] 2023/08/15 01:39:55.564585 [INF] Store Directory: "/tmp/opni-test-2132242568/jetstream/data/jetstream" + [231479] 2023/08/15 01:39:55.564589 [INF] ------------------------------------------- + [231479] 2023/08/15 01:39:55.565067 [INF] Listening for client connections on 0.0.0.0:35793 + [231479] 2023/08/15 01:39:55.565669 [INF] Server is ready + 2023-08-15T01:39:55Z INFO test.env Jetstream started + 2023-08-15T01:39:55Z DEBUG keyring loaded ephemeral key {"path": "/tmp/opni-test-2132242568/keyring/local-agent.json", "usage": "auth", "labels": {"opni.io/session-attribute":"local"}} + 2023-08-15T01:39:55Z DEBUG gateway found 0 plugins + 2023-08-15T01:39:55Z DEBUG gateway loaded plugin manifest {"plugins": 0} + 2023-08-15T01:39:55Z INFO gateway.cache compressing and archiving plugins... + 2023-08-15T01:39:55Z DEBUG gateway.cache added 0 new plugins to cache + 2023-08-15T01:39:55Z INFO gateway running plugin cache gc + 2023-08-15T01:39:55Z INFO gateway.update-server registering update handler for strategy "binary" + 2023-08-15T01:39:55Z INFO gateway.update-server registering update handler for strategy "kubernetes" + 2023-08-15T01:39:55Z INFO gateway.update-server registering update handler for strategy "noop" + 2023-08-15T01:39:55Z DEBUG gateway.grpc registering service {"service": "control.HealthListener"} + 2023-08-15T01:39:55Z DEBUG gateway.grpc registering internal service {"service": "stream.Delegate"} + 2023-08-15T01:39:55Z DEBUG gateway.grpc registering service {"service": "core.Pinger"} + 2023-08-15T01:39:55Z INFO test.env Loading gateway plugins... + 2023-08-15T01:39:55Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.System", "opni.apiextensions.HTTPAPIExtension", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.ManagementAPIExtension", "opni.backends.Capability", "opni.Metrics"]} + 2023-08-15T01:39:55Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.System", "opni.apiextensions.HTTPAPIExtension", "opni.apiextensions.ManagementAPIExtension", "opni.Metrics", "opni.backends.Capability", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.System"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.System"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/trace", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/profile", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/block", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/heap", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/ready", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/healthy", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG plugin.metrics/pkg/gateway.stream internal gateway stream connected + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service cortexadmin.CortexAdmin + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/metrics", "capability": "metrics"} + 2023-08-15T01:39:55Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/alerting", "capability": "alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_alerting"} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.AlertConditions + 2023-08-15T01:39:55Z DEBUG plugin.metrics/pkg/gateway.stream calling client handler + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "cortexadmin.CortexAdmin"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/AllUserStats"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/WriteMetrics"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/Query"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/QueryRange"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetRule"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricMetadata"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ListRules"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/LoadRules"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/DeleteRule"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/FlushBlocks"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetSeriesMetrics"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricLabelSets"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexConfig"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ExtractRawSeries"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "cortexadmin.CortexAdmin", "rules": 17} + 2023-08-15T01:39:55Z DEBUG gateway.grpc plugin rejected internal stream connection {"plugin": "plugin_alerting"} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "cortexadmin.CortexAdmin", "rules": [{"http":{"Pattern":{"Get":"/all_user_stats"}},"method":{"name":"AllUserStats","input_type":".google.protobuf.Empty","output_type":".cortexadmin.UserIDStatsList","options":{}}},{"http":{"Pattern":{"Post":"/write_metrics"},"body":"*"},"method":{"name":"WriteMetrics","input_type":".cortexadmin.WriteRequest","output_type":".cortexadmin.WriteResponse","options":{}}},{"http":{"Pattern":{"Get":"/query"},"additional_bindings":[{"Pattern":{"Post":"/query"},"body":"*"}]},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query"},"body":"*"},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/query_range"},"additional_bindings":[{"Pattern":{"Post":"/query_range"},"body":"*"}]},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query_range"},"body":"*"},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/rules/{namespace}/{groupName}"}},"method":{"name":"GetRule","input_type":".cortexadmin.GetRuleRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/metadata"}},"method":{"name":"GetMetricMetadata","input_type":".cortexadmin.MetricMetadataRequest","output_type":".cortexadmin.MetricMetadata","options":{}}},{"http":{"Pattern":{"Get":"/rules"}},"method":{"name":"ListRules","input_type":".cortexadmin.ListRulesRequest","output_type":".cortexadmin.ListRulesResponse","options":{}}},{"http":{"Pattern":{"Post":"/rules"}},"method":{"name":"LoadRules","input_type":".cortexadmin.LoadRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/rules/{groupName}"}},"method":{"name":"DeleteRule","input_type":".cortexadmin.DeleteRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/flush_blocks"}},"method":{"name":"FlushBlocks","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/series/metadata"}},"method":{"name":"GetSeriesMetrics","input_type":".cortexadmin.SeriesRequest","output_type":".cortexadmin.SeriesInfoList","options":{}}},{"http":{"Pattern":{"Get":"/series/labels"}},"method":{"name":"GetMetricLabelSets","input_type":".cortexadmin.LabelRequest","output_type":".cortexadmin.MetricLabels","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetCortexStatus","input_type":".google.protobuf.Empty","output_type":".cortexadmin.CortexStatus","options":{}}},{"http":{"Pattern":{"Get":"/config"}},"method":{"name":"GetCortexConfig","input_type":".cortexadmin.ConfigRequest","output_type":".cortexadmin.ConfigResponse","options":{}}},{"http":{"Pattern":{"Get":"/series/raw"}},"method":{"name":"ExtractRawSeries","input_type":".cortexadmin.MatcherRequest","output_type":".cortexadmin.QueryResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service cortexops.CortexOps + 2023-08-15T01:39:55Z INFO plugin.metrics.cortex-http configuring http api server + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "cortexops.CortexOps"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexops.CortexOps/ConfigureCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexops.CortexOps/UninstallCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "cortexops.CortexOps", "rules": 4} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "cortexops.CortexOps", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".cortexops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".cortexops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".cortexops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service remoteread.RemoteReadGateway + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z INFO plugin.alerting using cluster driver {"driver": "test-environment"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /ready", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z INFO plugin.opni.alerting starting opni embedded server {"addr": "127.0.0.1:43869"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "remoteread.RemoteReadGateway"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/AddTarget"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/EditTarget"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/RemoveTarget"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/ListTargets"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Start"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Stop"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/GetTargetStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Discover"} + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.AlertConditions"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionGroups"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CreateAlertCondition"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/GetAlertCondition"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditions"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/UpdateAlertCondition"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionChoices"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeleteAlertCondition"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "remoteread.RemoteReadGateway", "rules": 8} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/AlertConditionStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionsWithStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CloneTo"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ActivateSilence"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeactivateSilence"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/Timeline"} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "remoteread.RemoteReadGateway", "rules": [{"http":{"Pattern":{"Post":"/targets"},"body":"*"},"method":{"name":"AddTarget","input_type":".remoteread.TargetAddRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Put":"/targets"},"body":"*"},"method":{"name":"EditTarget","input_type":".remoteread.TargetEditRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/targets/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"RemoveTarget","input_type":".remoteread.TargetRemoveRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/targets"}},"method":{"name":"ListTargets","input_type":".remoteread.TargetListRequest","output_type":".remoteread.TargetList","options":{}}},{"http":{"Pattern":{"Post":"/targets/start/{target.meta.clusterId}/{target.meta.name}"},"body":"*"},"method":{"name":"Start","input_type":".remoteread.StartReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/targets/stop/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"Stop","input_type":".remoteread.StopReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status"},"body":"*"},"method":{"name":"GetTargetStatus","input_type":".remoteread.TargetStatusRequest","output_type":".remoteread.TargetStatus","options":{}}},{"http":{"Pattern":{"Post":"/edit"},"body":"*"},"method":{"name":"Discover","input_type":".remoteread.DiscoveryRequest","output_type":".remoteread.DiscoveryResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.AlertConditions", "rules": 13} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service node.metrics.NodeConfiguration + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.AlertConditions", "rules": [{"http":{"Pattern":{"Get":"/groups"}},"method":{"name":"ListAlertConditionGroups","input_type":".google.protobuf.Empty","output_type":".core.ReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertCondition","input_type":".alerting.AlertCondition","output_type":".alerting.ConditionReference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertCondition","input_type":".alerting.ConditionReference","output_type":".alerting.AlertCondition","options":{}}},{"http":{"Pattern":{"Post":"/list"},"body":"*"},"method":{"name":"ListAlertConditions","input_type":".alerting.ListAlertConditionRequest","output_type":".alerting.AlertConditionList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertCondition","input_type":".alerting.UpdateAlertConditionRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/choices"}},"method":{"name":"ListAlertConditionChoices","input_type":".alerting.AlertDetailChoicesRequest","output_type":".alerting.ListAlertTypeDetails","options":{}}},{"http":{"Pattern":{"Delete":"/configure"}},"method":{"name":"DeleteAlertCondition","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status/{id}"}},"method":{"name":"AlertConditionStatus","input_type":".alerting.ConditionReference","output_type":".alerting.AlertStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/list/withStatus"},"body":"*"},"method":{"name":"ListAlertConditionsWithStatus","input_type":".alerting.ListStatusRequest","output_type":".alerting.ListStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/clone"},"body":"*"},"method":{"name":"CloneTo","input_type":".alerting.CloneToRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/silences"},"body":"*"},"method":{"name":"ActivateSilence","input_type":".alerting.SilenceRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/silences"}},"method":{"name":"DeactivateSilence","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/timeline"}},"method":{"name":"Timeline","input_type":".alerting.TimelineRequest","output_type":".alerting.TimelineResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.AlertEndpoints + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.AlertEndpoints"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/CreateAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/GetAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ToggleNotifications"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ListAlertEndpoints"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/UpdateAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/DeleteAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/TestAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.AlertEndpoints", "rules": 7} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.AlertEndpoints", "rules": [{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertEndpoint","input_type":".alerting.AlertEndpoint","output_type":".core.Reference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertEndpoint","input_type":".core.Reference","output_type":".alerting.AlertEndpoint","options":{}}},{"http":{"Pattern":{"Post":"/toggleNotifications"},"body":"*"},"method":{"name":"ToggleNotifications","input_type":".alerting.ToggleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/list"}},"method":{"name":"ListAlertEndpoints","input_type":".alerting.ListAlertEndpointsRequest","output_type":".alerting.AlertEndpointList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertEndpoint","input_type":".alerting.UpdateAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/delete"},"body":"*"},"method":{"name":"DeleteAlertEndpoint","input_type":".alerting.DeleteAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/test"},"body":"*"},"method":{"name":"TestAlertEndpoint","input_type":".alerting.TestAlertEndpointRequest","output_type":".alerting.TestAlertEndpointResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.AlertNotifications + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.AlertNotifications"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/TriggerAlerts"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ResolveAlerts"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/PushNotification"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListNotifications"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListAlarmMessages"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListRoutingRelationships"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.AlertNotifications", "rules": 4} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.AlertNotifications", "rules": [{"http":{"Pattern":{"Post":"/trigger"},"body":"*"},"method":{"name":"TriggerAlerts","input_type":".alerting.TriggerAlertsRequest","output_type":".alerting.TriggerAlertsResponse","options":{}}},{"http":{"Pattern":{"Post":"/push"},"body":"*"},"method":{"name":"PushNotification","input_type":".alerting.Notification","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/notifications/list"},"body":"*"},"method":{"name":"ListNotifications","input_type":".alerting.ListNotificationRequest","output_type":".alerting.ListMessageResponse","options":{}}},{"http":{"Pattern":{"Post":"/alarms/list"},"body":"*"},"method":{"name":"ListAlarmMessages","input_type":".alerting.ListAlarmMessageRequest","output_type":".alerting.ListMessageResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.ops.AlertingAdmin + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "node.metrics.NodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "node.metrics.NodeConfiguration", "rules": 4} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "node.metrics.NodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".node.metrics.MetricsCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".node.metrics.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.ops.AlertingAdmin"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/ConfigureCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/InstallCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/UninstallCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/Info"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.ops.AlertingAdmin", "rules": 6} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.ops.AlertingAdmin", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".alerting.ops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".alerting.ops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/install"}},"method":{"name":"InstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".alerting.ops.UninstallRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/info"}},"method":{"name":"Info","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ComponentInfo","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.ops.ConfigReconciler + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.ConfigReconciler/SyncConfig"} + 2023-08-15T01:39:55Z INFO mgmt service has no http rules {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.node.AlertingNodeConfiguration + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.node.AlertingNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.node.AlertingNodeConfiguration", "rules": 4} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.node.AlertingNodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".alerting.node.AlertingCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".alerting.node.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.node.NodeAlertingCapability + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.node.NodeAlertingCapability"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.NodeAlertingCapability/Sync"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.node.NodeAlertingCapability", "rules": 1} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.node.NodeAlertingCapability", "rules": [{"http":{"Pattern":{"Post":"/v1/node/sync"},"body":"*"},"method":{"name":"Sync","input_type":".alerting.node.AlertingCapabilityConfig","output_type":".alerting.node.SyncResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO test.env Waiting for gateway to start... + 2023-08-15T01:39:55Z INFO test.env loaded 12 plugins + 2023-08-15T01:39:55Z INFO mgmt management HTTP server starting {"address": "localhost:39189"} + 2023-08-15T01:39:55Z INFO mgmt management gRPC server starting {"address": "127.0.0.1:42179"} + 2023-08-15T01:39:55Z INFO gateway.grpc gateway gRPC server starting {"address": "127.0.0.1:46029"} + 2023-08-15T01:39:55Z INFO gateway.http gateway HTTP server starting {"api": "127.0.0.1:37611", "metrics": "127.0.0.1:39213"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/tokens"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/tokens/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/watch"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/clusters/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/certs"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}/health"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/health/watch"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/clusters/{cluster.id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/roles"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/roles"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/roles/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/rolebindings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/rolebindings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/subjectaccess"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/apiextensions"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/capabilities"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/capabilities/{name}/installer"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/install"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/uninstall"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/cancel"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/dashboard/settings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/dashboard/settings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/all_user_stats"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/write_metrics"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules/{namespace}/{groupName}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/metadata"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/rules"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/CortexAdmin/rules/{groupName}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/flush_blocks"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/metadata"} + 2023-08-15T01:39:55Z INFO test.env Gateway started + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/labels"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/status"} +[BeforeSuite] PASSED [0.234 seconds] +------------------------------ + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/raw"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/configuration"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/uninstall"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/RemoteReadGateway/targets"} +agent capability tests when we use the alerting downstream capability should install the alerting cluster [integration] +/home/ec2-user/opni/test/plugins/alerting/capability_test.go:37 + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/RemoteReadGateway/targets/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:39:55Z INFO test.env Starting test environment + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/start/{target.meta.clusterId}/{target.meta.name}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/stop/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/edit"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertConditions/groups"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertConditions/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/choices"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/status/{id}"} + 2023-08-15T01:39:55Z INFO test.env Copying 7 files from embedded testdata/cortex to /tmp/opni-test-278561545/cortex + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/withStatus"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/clone"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/silences"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/silences"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/timeline"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/list/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/toggleNotifications"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertEndpoints/list"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/delete"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/test"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/trigger"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/push"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/notifications/list"} + 2023-08-15T01:39:55Z INFO test.env Waiting for jetstream to start... + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/alarms/list"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config/{node.id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/configuration"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/install"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/uninstall"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/info"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config/{node.id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/NodeAlertingCapability/v1/node/sync"} + [231489] 2023/08/15 01:39:55.799753 [INF] Starting nats-server + [231489] 2023/08/15 01:39:55.799817 [INF] Version: 2.9.21 + [231489] 2023/08/15 01:39:55.799822 [INF] Git: [b2e7725] + [231489] 2023/08/15 01:39:55.799834 [INF] Name: NDOMIA7P3BPCZ4AKM2VQ4E5SHQIQG7KFFF7UJKGLLKOMSPOHHVF362XI + [231489] 2023/08/15 01:39:55.799840 [INF] Node: 3KjorgZ4 + [231489] 2023/08/15 01:39:55.799843 [INF] ID: NDOMIA7P3BPCZ4AKM2VQ4E5SHQIQG7KFFF7UJKGLLKOMSPOHHVF362XI + [231489] 2023/08/15 01:39:55.799862 [INF] Using configuration file: /tmp/opni-test-278561545/jetstream/jetstream.conf + [231489] 2023/08/15 01:39:55.800154 [INF] Starting JetStream + [231489] 2023/08/15 01:39:55.800348 [INF] _ ___ _____ ___ _____ ___ ___ _ __ __ + [231489] 2023/08/15 01:39:55.800366 [INF] _ | | __|_ _/ __|_ _| _ \ __| /_\ | \/ | + [231489] 2023/08/15 01:39:55.800370 [INF] | || | _| | | \__ \ | | | / _| / _ \| |\/| | + [231489] 2023/08/15 01:39:55.800372 [INF] \__/|___| |_| |___/ |_| |_|_\___/_/ \_\_| |_| + [231489] 2023/08/15 01:39:55.800376 [INF] + [231489] 2023/08/15 01:39:55.800380 [INF] https://docs.nats.io/jetstream + [231489] 2023/08/15 01:39:55.800382 [INF] + [231489] 2023/08/15 01:39:55.800392 [INF] ---------------- JETSTREAM ---------------- + [231489] 2023/08/15 01:39:55.800399 [INF] Max Memory: 11.71 GB + [231489] 2023/08/15 01:39:55.800410 [INF] Max Storage: 5.29 GB + [231489] 2023/08/15 01:39:55.800419 [INF] Store Directory: "/tmp/opni-test-278561545/jetstream/data/jetstream" + [231489] 2023/08/15 01:39:55.800430 [INF] ------------------------------------------- + [231489] 2023/08/15 01:39:55.800894 [INF] Listening for client connections on 0.0.0.0:32947 + [231489] 2023/08/15 01:39:55.801237 [INF] Server is ready + 2023-08-15T01:39:55Z INFO test.env Jetstream started + 2023-08-15T01:39:55Z DEBUG keyring loaded ephemeral key {"path": "/tmp/opni-test-278561545/keyring/local-agent.json", "usage": "auth", "labels": {"opni.io/session-attribute":"local"}} + 2023-08-15T01:39:55Z DEBUG gateway found 0 plugins + 2023-08-15T01:39:55Z DEBUG gateway loaded plugin manifest {"plugins": 0} + 2023-08-15T01:39:55Z INFO gateway.cache compressing and archiving plugins... + 2023-08-15T01:39:55Z DEBUG gateway.cache added 0 new plugins to cache + 2023-08-15T01:39:55Z INFO gateway running plugin cache gc + 2023-08-15T01:39:55Z INFO gateway.update-server registering update handler for strategy "binary" + 2023-08-15T01:39:55Z INFO gateway.update-server registering update handler for strategy "kubernetes" + 2023-08-15T01:39:55Z INFO gateway.update-server registering update handler for strategy "noop" + 2023-08-15T01:39:55Z DEBUG gateway.grpc registering service {"service": "control.HealthListener"} + 2023-08-15T01:39:55Z DEBUG gateway.grpc registering internal service {"service": "stream.Delegate"} + 2023-08-15T01:39:55Z DEBUG gateway.grpc registering service {"service": "core.Pinger"} + 2023-08-15T01:39:55Z INFO test.env Loading gateway plugins... + 2023-08-15T01:39:55Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.System", "opni.apiextensions.HTTPAPIExtension", "opni.apiextensions.ManagementAPIExtension", "opni.Metrics", "opni.backends.Capability", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:39:55Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.Metrics", "opni.System", "opni.apiextensions.HTTPAPIExtension", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.ManagementAPIExtension", "opni.backends.Capability"]} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.System"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.System"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_metrics"} + 2023-08-15T01:39:55Z DEBUG plugin.metrics/pkg/gateway.stream internal gateway stream connected + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.AlertConditions + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/trace", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/profile", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/block", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/heap", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/ready", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/healthy", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/alerting", "capability": "alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.grpc plugin rejected internal stream connection {"plugin": "plugin_alerting"} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service cortexadmin.CortexAdmin + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG plugin.metrics/pkg/gateway.stream calling client handler + 2023-08-15T01:39:55Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/metrics", "capability": "metrics"} + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.AlertConditions"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionGroups"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CreateAlertCondition"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/GetAlertCondition"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditions"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/UpdateAlertCondition"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionChoices"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeleteAlertCondition"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/AlertConditionStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionsWithStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CloneTo"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ActivateSilence"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeactivateSilence"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/Timeline"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.AlertConditions", "rules": 13} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.AlertConditions", "rules": [{"http":{"Pattern":{"Get":"/groups"}},"method":{"name":"ListAlertConditionGroups","input_type":".google.protobuf.Empty","output_type":".core.ReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertCondition","input_type":".alerting.AlertCondition","output_type":".alerting.ConditionReference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertCondition","input_type":".alerting.ConditionReference","output_type":".alerting.AlertCondition","options":{}}},{"http":{"Pattern":{"Post":"/list"},"body":"*"},"method":{"name":"ListAlertConditions","input_type":".alerting.ListAlertConditionRequest","output_type":".alerting.AlertConditionList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertCondition","input_type":".alerting.UpdateAlertConditionRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/choices"}},"method":{"name":"ListAlertConditionChoices","input_type":".alerting.AlertDetailChoicesRequest","output_type":".alerting.ListAlertTypeDetails","options":{}}},{"http":{"Pattern":{"Delete":"/configure"}},"method":{"name":"DeleteAlertCondition","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status/{id}"}},"method":{"name":"AlertConditionStatus","input_type":".alerting.ConditionReference","output_type":".alerting.AlertStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/list/withStatus"},"body":"*"},"method":{"name":"ListAlertConditionsWithStatus","input_type":".alerting.ListStatusRequest","output_type":".alerting.ListStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/clone"},"body":"*"},"method":{"name":"CloneTo","input_type":".alerting.CloneToRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/silences"},"body":"*"},"method":{"name":"ActivateSilence","input_type":".alerting.SilenceRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/silences"}},"method":{"name":"DeactivateSilence","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/timeline"}},"method":{"name":"Timeline","input_type":".alerting.TimelineRequest","output_type":".alerting.TimelineResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.AlertEndpoints + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.AlertEndpoints"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/CreateAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/GetAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ToggleNotifications"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ListAlertEndpoints"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/UpdateAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/DeleteAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/TestAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.AlertEndpoints", "rules": 7} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.AlertEndpoints", "rules": [{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertEndpoint","input_type":".alerting.AlertEndpoint","output_type":".core.Reference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertEndpoint","input_type":".core.Reference","output_type":".alerting.AlertEndpoint","options":{}}},{"http":{"Pattern":{"Post":"/toggleNotifications"},"body":"*"},"method":{"name":"ToggleNotifications","input_type":".alerting.ToggleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/list"}},"method":{"name":"ListAlertEndpoints","input_type":".alerting.ListAlertEndpointsRequest","output_type":".alerting.AlertEndpointList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertEndpoint","input_type":".alerting.UpdateAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/delete"},"body":"*"},"method":{"name":"DeleteAlertEndpoint","input_type":".alerting.DeleteAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/test"},"body":"*"},"method":{"name":"TestAlertEndpoint","input_type":".alerting.TestAlertEndpointRequest","output_type":".alerting.TestAlertEndpointResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.AlertNotifications + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.AlertNotifications"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/TriggerAlerts"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ResolveAlerts"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/PushNotification"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListNotifications"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListAlarmMessages"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListRoutingRelationships"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.AlertNotifications", "rules": 4} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.AlertNotifications", "rules": [{"http":{"Pattern":{"Post":"/trigger"},"body":"*"},"method":{"name":"TriggerAlerts","input_type":".alerting.TriggerAlertsRequest","output_type":".alerting.TriggerAlertsResponse","options":{}}},{"http":{"Pattern":{"Post":"/push"},"body":"*"},"method":{"name":"PushNotification","input_type":".alerting.Notification","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/notifications/list"},"body":"*"},"method":{"name":"ListNotifications","input_type":".alerting.ListNotificationRequest","output_type":".alerting.ListMessageResponse","options":{}}},{"http":{"Pattern":{"Post":"/alarms/list"},"body":"*"},"method":{"name":"ListAlarmMessages","input_type":".alerting.ListAlarmMessageRequest","output_type":".alerting.ListMessageResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.ops.AlertingAdmin + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "cortexadmin.CortexAdmin"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/AllUserStats"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/WriteMetrics"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/Query"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/QueryRange"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetRule"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricMetadata"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ListRules"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/LoadRules"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/DeleteRule"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/FlushBlocks"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetSeriesMetrics"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricLabelSets"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexConfig"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ExtractRawSeries"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "cortexadmin.CortexAdmin", "rules": 17} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "cortexadmin.CortexAdmin", "rules": [{"http":{"Pattern":{"Get":"/all_user_stats"}},"method":{"name":"AllUserStats","input_type":".google.protobuf.Empty","output_type":".cortexadmin.UserIDStatsList","options":{}}},{"http":{"Pattern":{"Post":"/write_metrics"},"body":"*"},"method":{"name":"WriteMetrics","input_type":".cortexadmin.WriteRequest","output_type":".cortexadmin.WriteResponse","options":{}}},{"http":{"Pattern":{"Get":"/query"},"additional_bindings":[{"Pattern":{"Post":"/query"},"body":"*"}]},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query"},"body":"*"},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/query_range"},"additional_bindings":[{"Pattern":{"Post":"/query_range"},"body":"*"}]},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query_range"},"body":"*"},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/rules/{namespace}/{groupName}"}},"method":{"name":"GetRule","input_type":".cortexadmin.GetRuleRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/metadata"}},"method":{"name":"GetMetricMetadata","input_type":".cortexadmin.MetricMetadataRequest","output_type":".cortexadmin.MetricMetadata","options":{}}},{"http":{"Pattern":{"Get":"/rules"}},"method":{"name":"ListRules","input_type":".cortexadmin.ListRulesRequest","output_type":".cortexadmin.ListRulesResponse","options":{}}},{"http":{"Pattern":{"Post":"/rules"}},"method":{"name":"LoadRules","input_type":".cortexadmin.LoadRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/rules/{groupName}"}},"method":{"name":"DeleteRule","input_type":".cortexadmin.DeleteRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/flush_blocks"}},"method":{"name":"FlushBlocks","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/series/metadata"}},"method":{"name":"GetSeriesMetrics","input_type":".cortexadmin.SeriesRequest","output_type":".cortexadmin.SeriesInfoList","options":{}}},{"http":{"Pattern":{"Get":"/series/labels"}},"method":{"name":"GetMetricLabelSets","input_type":".cortexadmin.LabelRequest","output_type":".cortexadmin.MetricLabels","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetCortexStatus","input_type":".google.protobuf.Empty","output_type":".cortexadmin.CortexStatus","options":{}}},{"http":{"Pattern":{"Get":"/config"}},"method":{"name":"GetCortexConfig","input_type":".cortexadmin.ConfigRequest","output_type":".cortexadmin.ConfigResponse","options":{}}},{"http":{"Pattern":{"Get":"/series/raw"}},"method":{"name":"ExtractRawSeries","input_type":".cortexadmin.MatcherRequest","output_type":".cortexadmin.QueryResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service cortexops.CortexOps + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.ops.AlertingAdmin"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/ConfigureCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/InstallCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/UninstallCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/Info"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.ops.AlertingAdmin", "rules": 6} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.ops.AlertingAdmin", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".alerting.ops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".alerting.ops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/install"}},"method":{"name":"InstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".alerting.ops.UninstallRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/info"}},"method":{"name":"Info","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ComponentInfo","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.ops.ConfigReconciler + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.ConfigReconciler/SyncConfig"} + 2023-08-15T01:39:55Z INFO mgmt service has no http rules {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:39:55Z INFO plugin.alerting using cluster driver {"driver": "test-environment"} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.node.AlertingNodeConfiguration + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.node.AlertingNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.node.AlertingNodeConfiguration", "rules": 4} + 2023-08-15T01:39:55Z INFO plugin.opni.alerting starting opni embedded server {"addr": "127.0.0.1:35689"} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.node.AlertingNodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".alerting.node.AlertingCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".alerting.node.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.node.NodeAlertingCapability + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.node.NodeAlertingCapability"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.NodeAlertingCapability/Sync"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.node.NodeAlertingCapability", "rules": 1} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.node.NodeAlertingCapability", "rules": [{"http":{"Pattern":{"Post":"/v1/node/sync"},"body":"*"},"method":{"name":"Sync","input_type":".alerting.node.AlertingCapabilityConfig","output_type":".alerting.node.SyncResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO plugin.metrics.cortex-http configuring http api server + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "cortexops.CortexOps"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexops.CortexOps/ConfigureCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexops.CortexOps/UninstallCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "cortexops.CortexOps", "rules": 4} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "cortexops.CortexOps", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".cortexops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".cortexops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".cortexops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service remoteread.RemoteReadGateway + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /ready", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "remoteread.RemoteReadGateway"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/AddTarget"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/EditTarget"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/RemoveTarget"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/ListTargets"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Start"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Stop"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/GetTargetStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Discover"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "remoteread.RemoteReadGateway", "rules": 8} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "remoteread.RemoteReadGateway", "rules": [{"http":{"Pattern":{"Post":"/targets"},"body":"*"},"method":{"name":"AddTarget","input_type":".remoteread.TargetAddRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Put":"/targets"},"body":"*"},"method":{"name":"EditTarget","input_type":".remoteread.TargetEditRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/targets/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"RemoveTarget","input_type":".remoteread.TargetRemoveRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/targets"}},"method":{"name":"ListTargets","input_type":".remoteread.TargetListRequest","output_type":".remoteread.TargetList","options":{}}},{"http":{"Pattern":{"Post":"/targets/start/{target.meta.clusterId}/{target.meta.name}"},"body":"*"},"method":{"name":"Start","input_type":".remoteread.StartReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/targets/stop/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"Stop","input_type":".remoteread.StopReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status"},"body":"*"},"method":{"name":"GetTargetStatus","input_type":".remoteread.TargetStatusRequest","output_type":".remoteread.TargetStatus","options":{}}},{"http":{"Pattern":{"Post":"/edit"},"body":"*"},"method":{"name":"Discover","input_type":".remoteread.DiscoveryRequest","output_type":".remoteread.DiscoveryResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service node.metrics.NodeConfiguration + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "node.metrics.NodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "node.metrics.NodeConfiguration", "rules": 4} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "node.metrics.NodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".node.metrics.MetricsCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".node.metrics.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:39:55Z INFO test.env Waiting for gateway to start... + 2023-08-15T01:39:55Z INFO test.env loaded 12 plugins + 2023-08-15T01:39:55Z INFO gateway.grpc gateway gRPC server starting {"address": "127.0.0.1:38891"} + 2023-08-15T01:39:55Z INFO gateway.http gateway HTTP server starting {"api": "127.0.0.1:43103", "metrics": "127.0.0.1:36995"} + 2023-08-15T01:39:55Z INFO mgmt management HTTP server starting {"address": "localhost:34163"} + 2023-08-15T01:39:55Z INFO mgmt management gRPC server starting {"address": "127.0.0.1:46361"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/tokens"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/tokens/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/watch"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/clusters/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/certs"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}/health"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/health/watch"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/clusters/{cluster.id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/roles"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/roles"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/roles/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/rolebindings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/rolebindings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/subjectaccess"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/apiextensions"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/capabilities"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/capabilities/{name}/installer"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/install"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/uninstall"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/cancel"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/dashboard/settings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/dashboard/settings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertConditions/groups"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertConditions/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/choices"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/status/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/withStatus"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/clone"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/silences"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/silences"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/timeline"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/list/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/toggleNotifications"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertEndpoints/list"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/delete"} + 2023-08-15T01:39:55Z INFO test.env Gateway started + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/test"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/trigger"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/push"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/notifications/list"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/alarms/list"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/all_user_stats"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/write_metrics"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules/{namespace}/{groupName}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/metadata"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/rules"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/CortexAdmin/rules/{groupName}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/flush_blocks"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/metadata"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/labels"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/raw"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/configuration"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/install"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/uninstall"} + 2023-08-15T01:39:55Z DEBUG plugin.alerting-test-cluster-driver Syncer start : alerting-server --syncer.alertmanager.config.file=/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml --syncer.listen.address=:45391 --syncer.alertmanager.address=http://127.0.0.1:33771 --syncer.gateway.join.address=:46361 syncer {"config-file": "/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/info"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:39:55Z INFO plugin.alerting-test-cluster-driver Starting AlertManager {"config-file": "/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml", "alertmanager-port": 33771, "opni-port": 41223} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config/{node.id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/NodeAlertingCapability/v1/node/sync"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/configuration"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/uninstall"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/RemoteReadGateway/targets/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/start/{target.meta.clusterId}/{target.meta.name}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/stop/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/edit"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config/{node.id}"} + ts=2023-08-15T01:39:56.082Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:39:56.082Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + ts=2023-08-15T01:39:56.085Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=44749 + ts=2023-08-15T01:39:56.086Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + ts=2023-08-15T01:39:56.116Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml + ts=2023-08-15T01:39:56.117Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml + ts=2023-08-15T01:39:56.119Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:33771 + ts=2023-08-15T01:39:56.119Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:33771 + 2023-08-15T01:39:56Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:39:56Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:39:56Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:39:56Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:39:56Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:39:56Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:39:56Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:39:56Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:39:56Z INFO plugin.alerting-test-cluster-driver Starting AlertManager Syncer {"config-file": "/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml", "port": 45391} + 2023-08-15T01:39:56Z INFO plugin.alerting-test-cluster-driver AlertManager started {"config-file": "/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml", "address": "http://127.0.0.1:33771"} + 2023-08-15T01:39:56Z INFO plugin.alerting updating known peers : [{http://127.0.0.1:33771 http://127.0.0.1:35689}] +• [1.209 seconds] +------------------------------ +agent capability tests when we use the alerting downstream capability should be able to install the alerting capability on all downstream agents [integration] +/home/ec2-user/opni/test/plugins/alerting/capability_test.go:60 + STEP: bootstrapping agents @ 08/15/23 01:39:56.998 + 2023-08-15T01:39:57Z INFO test starting agent {"id": "agent1", "address": "127.0.0.1:43003", "version": "v2"} + 2023-08-15T01:39:57Z DEBUG agent using log level: debug + 2023-08-15T01:39:57Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:39:57Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent1 + 2023-08-15T01:39:57Z INFO agent bootstrap completed successfully + 2023-08-15T01:39:57Z WARN jetstream disconnected from jetstream + 2023-08-15T01:39:57Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:39:57Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent1 is now active + 2023-08-15T01:39:57Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent1 is now active + 2023-08-15T01:39:57Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG commands/alerting.go:72 syncer gateway join address:46361 + 2023-08-15T01:39:57Z INFO plugin.alerting-syncer syncer/syncer_server.go:90 starting alerting syncer server as identity 2322db02-3463-4d35-bdf7-078193fc2f5e + 2023-08-15T01:39:57Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.apiextensions.HTTPAPIExtension", "opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:39:57Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:108 acquired gateway alerting client + 2023-08-15T01:39:57Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:39:57Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:144 starting alerting syncer stream subscription + 2023-08-15T01:39:57Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:39:57Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "4d74f48a-1c76-4e7f-a440-257aa59e8ac1"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:150 starting new stream subscription + 2023-08-15T01:39:57Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:39:57Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:39:57Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "447ca987-ca2d-4c53-868b-50730809d173"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:39:57Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:39:57Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:39:57Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:39:57Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:39:57Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:39:57Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:39:57Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:39:57Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:39:57Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:39:57Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:39:57Z INFO agent using unmanaged plugin loader + 2023-08-15T01:39:57Z INFO agent agent http server starting {"address": "127.0.0.1:43003"} + 2023-08-15T01:39:57Z INFO agent connecting to gateway... + 2023-08-15T01:39:57Z DEBUG gateway ratelimit: 50.000000 available + 2023-08-15T01:39:57Z INFO test starting agent {"id": "agent2", "address": "127.0.0.1:33707", "version": "v2"} + 2023-08-15T01:39:57Z DEBUG agent using log level: debug + 2023-08-15T01:39:57Z DEBUG gateway-client authenticated {"id": "agent1"} + 2023-08-15T01:39:57Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:39:57Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml"} + 2023-08-15T01:39:57Z DEBUG gateway.grpc handling new stream connection + ts=2023-08-15T01:39:57.110Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml + ts=2023-08-15T01:39:57.111Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:39:57Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent1"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream stream connected + 2023-08-15T01:39:57Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent1"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:39:57Z DEBUG gateway.sync agent connected {"id": "agent1"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:39:57Z INFO gateway.monitor received status update {"id": "agent1", "connected": true} + 2023-08-15T01:39:57Z INFO agent bootstrap completed successfully + 2023-08-15T01:39:57Z DEBUG gateway.delegate agent connected {"id": "agent1"} + 2023-08-15T01:39:57Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent2 + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:39:57Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:39:57Z WARN jetstream disconnected from jetstream + 2023-08-15T01:39:57Z INFO agent gateway connected + 2023-08-15T01:39:57Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:39:57Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent2 is now active + 2023-08-15T01:39:57Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z INFO gateway.monitor received health update {"id": "agent1", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:39:57Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.HTTPAPIExtension"]} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent1", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:39:57Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent2 is now active + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:39:57Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:39:57Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:39:57Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:39:57Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:39:57Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:39:57Z INFO gateway.monitor received health update {"id": "agent1", "ready": true, "conditions": []} + 2023-08-15T01:39:57Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:39:57Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:39:57Z DEBUG test.env configuring node {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:39:57Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:39:57Z DEBUG test.env configuring node {"node": "agent1", "driver": "otel"} + 2023-08-15T01:39:57Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "otel"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent1"} + 2023-08-15T01:39:57Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:39:57Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:39:57Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:39:57Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:39:57Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:39:57Z INFO agent using unmanaged plugin loader + 2023-08-15T01:39:57Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:39:57Z INFO agent agent http server starting {"address": "127.0.0.1:33707"} + 2023-08-15T01:39:57Z INFO agent connecting to gateway... + 2023-08-15T01:39:57Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:39:57Z DEBUG gateway ratelimit: 49.952639 available + 2023-08-15T01:39:57Z INFO test starting agent {"id": "agent3", "address": "127.0.0.1:40781", "version": "v2"} + 2023-08-15T01:39:57Z DEBUG gateway-client authenticated {"id": "agent2"} + 2023-08-15T01:39:57Z DEBUG agent using log level: debug + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:39:57Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG gateway.grpc handling new stream connection + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:39:57Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent2"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream stream connected + 2023-08-15T01:39:57Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:39:57Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:39:57Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting rule sync is disabled {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent2"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG gateway.sync agent connected {"id": "agent2"} + 2023-08-15T01:39:57Z INFO gateway.monitor received status update {"id": "agent2", "connected": true} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:39:57Z DEBUG gateway.delegate agent connected {"id": "agent2"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:39:57Z INFO gateway.monitor received health update {"id": "agent2", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:39:57Z INFO agent gateway connected + 2023-08-15T01:39:57Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent3 + 2023-08-15T01:39:57Z INFO agent bootstrap completed successfully + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:39:57Z WARN jetstream disconnected from jetstream + 2023-08-15T01:39:57Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:39:57Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:39:57Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent3 is now active + 2023-08-15T01:39:57Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.HTTPAPIExtension"]} + 2023-08-15T01:39:57Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent2", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent3 is now active + 2023-08-15T01:39:57Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:39:57Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent2"} + 2023-08-15T01:39:57Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:39:57Z DEBUG test.env configuring node {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:39:57Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:39:57Z DEBUG test.env nothing to do (already stopped) {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:39:57Z DEBUG test.env configuring node {"node": "agent2", "driver": "otel"} + 2023-08-15T01:39:57Z DEBUG test.env nothing to do (already stopped) {"node": "agent2", "driver": "otel"} + 2023-08-15T01:39:57Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:39:57Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:39:57Z INFO gateway.monitor received health update {"id": "agent2", "ready": true, "conditions": []} + 2023-08-15T01:39:57Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:39:57Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:39:57Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:39:57Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:39:57Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:39:57Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:39:57Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:39:57Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:39:57Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:39:57Z INFO agent using unmanaged plugin loader + 2023-08-15T01:39:57Z INFO agent agent http server starting {"address": "127.0.0.1:40781"} + 2023-08-15T01:39:57Z INFO agent connecting to gateway... + 2023-08-15T01:39:57Z DEBUG gateway ratelimit: 49.942679 available + 2023-08-15T01:39:57Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent1", "capabilities": ["alerting"]} + 2023-08-15T01:39:57Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:39:57Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG gateway-client authenticated {"id": "agent3"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG gateway.grpc handling new stream connection + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream stream connected + 2023-08-15T01:39:57Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "agent1", "capability": "alerting"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent2", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent2", "capabilities": ["alerting"]} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent3"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:39:57Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:39:57Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "agent2", "capability": "alerting"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting rule sync is disabled {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent3"} + 2023-08-15T01:39:57Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent2", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting starting rule stream {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:39:57Z INFO plugin.alerting waiting for rule sync client... {"component": "rule-streamer"} + 2023-08-15T01:39:57Z INFO plugin.alerting rule sync client acquired {"component": "rule-streamer"} + 2023-08-15T01:39:57Z INFO plugin.alerting discovered 1 rules {"component": "rule-streamer"} + 2023-08-15T01:39:57Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "agent3", "capability": "alerting"} + 2023-08-15T01:39:57Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting starting rule stream {"component": "rule-streamer", "nodeId": "agent2"} +• [0.332 seconds] +------------------------------ + 2023-08-15T01:39:57Z INFO plugin.alerting waiting for rule sync client... {"component": "rule-streamer"} + 2023-08-15T01:39:57Z INFO plugin.alerting rule sync client acquired {"component": "rule-streamer"} + 2023-08-15T01:39:57Z INFO plugin.alerting discovered 1 rules {"component": "rule-streamer"} +agent capability tests when we use the alerting downstream capability should keep track of the downstream synced alerting groups [integration] +/home/ec2-user/opni/test/plugins/alerting/capability_test.go:92 + 2023-08-15T01:39:57Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:39:57Z DEBUG gateway.sync agent connected {"id": "agent3"} + 2023-08-15T01:39:57Z INFO gateway.monitor received status update {"id": "agent3", "connected": true} + 2023-08-15T01:39:57Z DEBUG gateway.delegate agent connected {"id": "agent3"} + 2023-08-15T01:39:57Z INFO gateway.monitor received health update {"id": "agent3", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:39:57Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z INFO agent gateway connected + 2023-08-15T01:39:57Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent3", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:39:57Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent3"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting starting rule stream {"component": "rule-streamer", "nodeId": "agent3"} + 2023-08-15T01:39:57Z INFO gateway.monitor received health update {"id": "agent3", "ready": true, "conditions": []} + 2023-08-15T01:39:57Z INFO plugin.alerting waiting for rule sync client... {"component": "rule-streamer"} + 2023-08-15T01:39:57Z INFO plugin.alerting rule sync client acquired {"component": "rule-streamer"} + 2023-08-15T01:39:57Z INFO plugin.alerting discovered 1 rules {"component": "rule-streamer"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:39:57Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:39:57Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent3", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:39:57Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:39:57Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent3"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:39:57Z DEBUG test.env configuring node {"node": "agent3", "driver": "otel"} + 2023-08-15T01:39:57Z DEBUG test.env nothing to do (already stopped) {"node": "agent3", "driver": "otel"} + 2023-08-15T01:39:57Z DEBUG test.env configuring node {"node": "agent3", "driver": "prometheus"} + 2023-08-15T01:39:57Z DEBUG test.env nothing to do (already stopped) {"node": "agent3", "driver": "prometheus"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:39:57Z DEBUG plugin.metrics sent node health update +• [0.214 seconds] +------------------------------ +agent capability tests when we use the alerting downstream capability should have synced downstream prometheus alerting rules [integration] +/home/ec2-user/opni/test/plugins/alerting/capability_test.go:107 +• [0.033 seconds] +------------------------------ +agent capability tests when we use the alerting downstream capability should be able to manipulate a sub-set of the synced rule configuration [integration] +/home/ec2-user/opni/test/plugins/alerting/capability_test.go:141 + 2023-08-15T01:39:57Z DEBUG plugin.alerting Updating alert condition id:"test-rule-16578271-d08c-4b36-b12e-e8026220a9b3" groupId:"test-group-16578271-d08c-4b36-b12e-e8026220a9b3" {"component": "alarms", "handler": "UpdateAlertCondition"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting Updating alert condition id:"test-rule-944f7e8d-6a94-4f4d-92a4-55b77d724d8a" groupId:"test-group-944f7e8d-6a94-4f4d-92a4-55b77d724d8a" {"component": "alarms", "handler": "UpdateAlertCondition"} + 2023-08-15T01:39:57Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting Updating alert condition id:"test-rule-eb475fbb-4d41-48d1-8a4c-5ceca3228273" groupId:"test-group-eb475fbb-4d41-48d1-8a4c-5ceca3228273" {"component": "alarms", "handler": "UpdateAlertCondition"} + 2023-08-15T01:39:57Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:39:57Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:39:57Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} +• [0.042 seconds] +------------------------------ +agent capability tests when we use the alerting downstream capability should be able to uninstall the alerting capability on all downstream agents [integration] +/home/ec2-user/opni/test/plugins/alerting/capability_test.go:186 + 2023-08-15T01:39:57Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent1", "capabilities": ["alerting"]} + 2023-08-15T01:39:57Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:39:57Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "agent1", "capability": "alerting"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent2", "capabilities": ["alerting"]} + 2023-08-15T01:39:57Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting stopping rule stream {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:39:57Z INFO plugin.alerting Exiting rule sync loop {"component": "rule-streamer"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:39:57Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "agent2", "capability": "alerting"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent2", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent3", "capabilities": ["alerting"]} + 2023-08-15T01:39:57Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting stopping rule stream {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z INFO plugin.alerting Exiting rule sync loop {"component": "rule-streamer"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:39:57Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "agent3", "capability": "alerting"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent3", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "447ca987-ca2d-4c53-868b-50730809d173"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "4d74f48a-1c76-4e7f-a440-257aa59e8ac1"} + 2023-08-15T01:39:57Z INFO plugin.alerting shutting down cluster driver update handler + 2023-08-15T01:39:57Z INFO plugin.alerting exiting main sync loop + 2023-08-15T01:39:57Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:39:57Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:39:57Z INFO plugin.alerting condition set {"condition": "Config Sync", "status": "Failure", "reason": "error syncing alerting node : context canceled"} + 2023-08-15T01:39:57Z ERROR test agent "agent1" exited: context canceled + 2023-08-15T01:39:57Z DEBUG plugin.alerting closing cortex cluster status watcher... {"watcher": "cortex-cluster-status"} + 2023-08-15T01:39:57Z ERROR plugin.alerting failed to receive cluster event : rpc error: code = Canceled desc = context canceled + 2023-08-15T01:39:57Z WARN plugin.alerting failed to send %s node health updatesalerting {"component": "health-cfg-sync", "err": "context canceled"} + 2023-08-15T01:39:57Z WARN jetstream disconnected from jetstream + 2023-08-15T01:39:57Z INFO gateway shutting down plugins + 2023-08-15T01:39:57Z INFO plugin.alerting Exiting rule sync loop {"component": "rule-streamer"} + 2023-08-15T01:39:57Z WARN plugin.alerting failed to receive cluster health status from grpc stream, retrying... +2023-08-15T01:39:57.654Z WARN totem.gateway-server stream.go:231 stream error; kicking {"error": "rpc error: code = Unavailable desc = transport is closing"} + 2023-08-15T01:39:57Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:39:57Z WARN jetstream disconnected from jetstream + 2023-08-15T01:39:57Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:39:57Z DEBUG gateway.delegate agent disconnected {"id": "agent1"} + 2023-08-15T01:39:57Z INFO gateway.grpc agent stream closing {"error": "context canceled"} + 2023-08-15T01:39:57Z DEBUG gateway.sync agent disconnected {"id": "agent1"} + 2023-08-15T01:39:57.655Z WARN totem.gateway-server server.go:222 stream exited with error {"error": "rpc error: code = Unavailable desc = transport is closing"} +2023-08-15T01:39:57Z INFO gateway all plugins shut down + 2023-08-15T01:39:57Z WARN jetstream disconnected from jetstream + 2023-08-15T01:39:57Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:39:57Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:39:57Z DEBUG gateway.delegate agent disconnected {"id": "agent2"} + 2023-08-15T01:39:57Z DEBUG gateway.sync agent disconnected {"id": "agent2"} + 2023-08-15T01:39:57Z INFO gateway.grpc agent stream closing {"error": "context canceled"} +2023-08-15T01:39:57.655Z WARN totem.gateway-server.gateway-server->plugin_metrics.gateway-server->plugin_metrics stream.go:231 stream error; kicking {"error": "rpc error: code = Unavailable desc = transport is closing"} + 2023-08-15T01:39:57Z WARN gateway http server exited with error {"error": "context canceled"} +2023-08-15T01:39:57.657Z WARN totem.gateway-server server.go:154 stream exited with error {"error": "rpc error: code = Unavailable desc = transport is closing"} + 2023-08-15T01:39:57Z ERROR test agent "agent3" exited: context canceled + 2023-08-15T01:39:57Z ERROR test agent "agent2" exited: context canceled + 2023-08-15T01:39:57Z WARN test.env management server exited with error {"error": "context canceled"} + 2023-08-15T01:39:57Z DEBUG gateway.delegate agent disconnected {"id": "agent3"} + 2023-08-15T01:39:57Z WARN test.env gateway server exited with error {"error": "context canceled"} + 2023-08-15T01:39:57Z INFO gateway.grpc agent stream closing {"error": "context canceled"} + 2023-08-15T01:39:57Z DEBUG gateway.sync agent disconnected {"id": "agent3"} + 2023-08-15T01:39:57Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} +2023-08-15T01:39:57.658Z WARN totem.gateway-server.gateway-server->plugin_alerting.gateway-server->plugin_alerting stream.go:231 stream error; kicking {"error": "rpc error: code = Unavailable desc = transport is closing"} + ts=2023-08-15T01:39:57.651Z caller=alertmanager_main.go:601 level=info msg="Received SIGTERM, exiting gracefully..." + ts=2023-08-15T01:39:57.652Z caller=cluster.go:690 level=info component=cluster msg="gossip not settled but continuing anyway" polls=0 elapsed=1.565609031s + 2023-08-15T01:39:57Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:39:57Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:39:57Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:39:57Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error + 2023-08-15T01:39:57Z ERROR plugin.alerting EOF + 2023-08-15T01:39:57Z ERROR plugin.alerting failed to get node capability spec {"component": "node-backend", "error": "rpc error: code = Unknown desc = nats: connection closed"} + 2023-08-15T01:39:57Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:39:57Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:39:57Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error + 2023-08-15T01:39:57Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:39:57Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error + 2023-08-15T01:39:57Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error + 2023-08-15T01:39:57Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} +• [0.044 seconds] +------------------------------ +Alerting routing logic translation to physical dispatching when setting namespace specs on the routing tree should be able to dynamically update alert routing [integration] +/home/ec2-user/opni/test/plugins/alerting/routing_test.go:77 + STEP: Creating some test webhook servers @ 08/15/23 01:39:57.667 + STEP: routing to a subset of the test webhook servers @ 08/15/23 01:39:57.668 + STEP: setting the router to the namespace specs @ 08/15/23 01:39:57.668 + STEP: running alertmanager with this config @ 08/15/23 01:39:57.668 + STEP: expecting to successfully build a config @ 08/15/23 01:39:57.669 + STEP: checking that the config passes the marshalling validation @ 08/15/23 01:39:57.669 + STEP: checking that the config passes the unmarshalling validation @ 08/15/23 01:39:57.673 + STEP: writing the config file to a temporary directory @ 08/15/23 01:39:57.678 + STEP: Verifying that the config can be loaded by alertmanager @ 08/15/23 01:39:57.679 + 2023-08-15T01:39:57Z INFO test.env Waiting for alertmanager to start... + ts=2023-08-15T01:39:57.778Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:39:57.778Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + ts=2023-08-15T01:39:57.778Z caller=cluster.go:173 level=debug component=cluster msg="resolved peers to following addresses" peers= + 2023-08-15T01:39:57Z INFO plugin.opni.alerting extensions/embedded_server.go:88 starting opni embedded server {"addr": ":37795"} + ts=2023-08-15T01:39:57.782Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=36609 + ts=2023-08-15T01:39:57.782Z caller=delegate.go:236 level=debug component=cluster received=NotifyJoin node=01H7VDK16PBHAH6DXXGH2DBC9M addr=172.31.7.66:36609 + ts=2023-08-15T01:39:57.782Z caller=nflog.go:275 level=debug component=nflog msg="notification log snapshot file doesn't exist" err="open /tmp/opni-test-2132242568/alertmanager_data/496233c9-b52c-41ec-873b-750058418da5/nflog: no such file or directory" + ts=2023-08-15T01:39:57.783Z caller=silence.go:353 level=debug component=silences msg="silences snapshot file doesn't exist" err="open /tmp/opni-test-2132242568/alertmanager_data/496233c9-b52c-41ec-873b-750058418da5/silences: no such file or directory" + ts=2023-08-15T01:39:57.783Z caller=cluster.go:266 level=debug component=cluster msg="joined cluster" peers=0 + ts=2023-08-15T01:39:57.783Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + ts=2023-08-15T01:39:57.816Z caller=alertmanager_main.go:413 level=debug externalURL=http://ip-172-31-7-66.us-east-2.compute.internal:45495 + ts=2023-08-15T01:39:57.816Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/webhook-4e35a247-5d11-489e-83c8-40fa597e3e97/config-FFdQc2CufUZM7fS4V7Hxkj + ts=2023-08-15T01:39:57.817Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/webhook-4e35a247-5d11-489e-83c8-40fa597e3e97/config-FFdQc2CufUZM7fS4V7Hxkj + ts=2023-08-15T01:39:57.818Z caller=alertmanager_main.go:543 level=debug routePrefix=/ + ts=2023-08-15T01:39:57.819Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:45495 + ts=2023-08-15T01:39:57.819Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:45495 + 2023-08-15T01:39:58Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:39:58Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:39:58Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:39:58Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:39:58Z INFO test.env AlertManager started {"address": "http://localhost:45495", "opni-address": "http://localhost:824683462744"} + STEP: sending alerts to each condition in the router @ 08/15/23 01:39:58.684 + ts=2023-08-15T01:39:58.685Z caller=dispatch.go:163 level=debug component=dispatcher msg="Received alert" alert=[f5800cc][active] + ts=2023-08-15T01:39:58.686Z caller=dispatch.go:163 level=debug component=dispatcher msg="Received alert" alert=[843c13a][active] + ts=2023-08-15T01:39:58.686Z caller=dispatch.go:163 level=debug component=dispatcher msg="Received alert" alert=[b92943f][active] + STEP: getting the AlertManager state @ 08/15/23 01:39:58.687 + 2023-08-15T01:39:59Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:39:59Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:39:59Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:39:59Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + ts=2023-08-15T01:39:59.686Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test!=\"\"}:{opni_uuid=\"0492e20c-7f62-45e6-a1df-458561ea691e\", test=\"0492e20c-7f62-45e6-a1df-458561ea691e\"}" msg=flushing alerts=[[f5800cc][active]] + ts=2023-08-15T01:39:59.686Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}:{opni_uuid=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\", test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}" msg=flushing alerts=[[843c13a][active]] + ts=2023-08-15T01:39:59.686Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test!=\"\"}:{opni_uuid=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\", test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}" msg=flushing alerts=[[843c13a][active]] + ts=2023-08-15T01:39:59.686Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"0492e20c-7f62-45e6-a1df-458561ea691e\"}:{opni_uuid=\"0492e20c-7f62-45e6-a1df-458561ea691e\", test=\"0492e20c-7f62-45e6-a1df-458561ea691e\"}" msg=flushing alerts=[[f5800cc][active]] + ts=2023-08-15T01:39:59.687Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}:{opni_uuid=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\", test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}" msg=flushing alerts=[[b92943f][active]] + ts=2023-08-15T01:39:59.687Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test!=\"\"}:{opni_uuid=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\", test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}" msg=flushing alerts=[[b92943f][active]] + STEP: getting the AlertManager state @ 08/15/23 01:39:59.691 + ts=2023-08-15T01:39:59.784Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.001099747s + 2023-08-15T01:40:00Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:00Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:00Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:00Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:00.694 + 2023-08-15T01:40:01Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:01Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:01Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:01Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:01.698 + ts=2023-08-15T01:40:01.785Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=4.002260686s + 2023-08-15T01:40:02Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:02Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:02Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:02Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:02.702 + 2023-08-15T01:40:03Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:03Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:03Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:03Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:03.706 + ts=2023-08-15T01:40:03.786Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=6.003212044s + 2023-08-15T01:40:04Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:04Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:04Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:04Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:04.71 + 2023-08-15T01:40:05Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:05Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:05Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:05Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:05.714 + ts=2023-08-15T01:40:05.787Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=8.003730932s + 2023-08-15T01:40:06Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:06Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:06Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:06Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:06.718 + 2023-08-15T01:40:07Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:07Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:07Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:07Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:07.721 + ts=2023-08-15T01:40:07.788Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.004699276s + ts=2023-08-15T01:40:07.792Z caller=notify.go:751 level=debug component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:07.792Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__0492e20c-7f62-45e6-a1df-458561ea691e integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:07.792Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__a135cbb4-f039-4f29-9153-dfe0ae62dd32 integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:07.792Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__22e67d2a-f6c0-461e-8a14-ee6be7004cfd integration=webhook[1] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:07.792Z caller=notify.go:751 level=debug component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:07.792Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__22e67d2a-f6c0-461e-8a14-ee6be7004cfd integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:07.792Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__a135cbb4-f039-4f29-9153-dfe0ae62dd32 integration=webhook[2] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:07.792Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"0492e20c-7f62-45e6-a1df-458561ea691e\"}:{opni_uuid=\"0492e20c-7f62-45e6-a1df-458561ea691e\", test=\"0492e20c-7f62-45e6-a1df-458561ea691e\"}" msg=flushing alerts=[[f5800cc][active]] + ts=2023-08-15T01:40:07.792Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}:{opni_uuid=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\", test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}" msg=flushing alerts=[[843c13a][active]] + ts=2023-08-15T01:40:07.792Z caller=notify.go:751 level=debug component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:07.793Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__a135cbb4-f039-4f29-9153-dfe0ae62dd32 integration=webhook[1] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:07.793Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}:{opni_uuid=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\", test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}" msg=flushing alerts=[[b92943f][active]] + 2023-08-15T01:40:08Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:08Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:08Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:08Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:08.725 + STEP: deleting a random server endpoint @ 08/15/23 01:40:08.728 + STEP: expecting to successfully build a config @ 08/15/23 01:40:08.729 + STEP: checking that the config passes the marshalling validation @ 08/15/23 01:40:08.729 + STEP: checking that the config passes the unmarshalling validation @ 08/15/23 01:40:08.732 + STEP: writing the config file to a temporary directory @ 08/15/23 01:40:08.738 + STEP: Verifying that the config can be loaded by alertmanager @ 08/15/23 01:40:08.738 + 2023-08-15T01:40:08Z INFO test.env Waiting for alertmanager to start... + ts=2023-08-15T01:40:08.836Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:40:08.837Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + ts=2023-08-15T01:40:08.837Z caller=cluster.go:173 level=debug component=cluster msg="resolved peers to following addresses" peers= + 2023-08-15T01:40:08Z INFO plugin.opni.alerting extensions/embedded_server.go:88 starting opni embedded server {"addr": ":45149"} + ts=2023-08-15T01:40:08.839Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=40461 + ts=2023-08-15T01:40:08.840Z caller=delegate.go:236 level=debug component=cluster received=NotifyJoin node=01H7VDKC07RJNFMDS3ZZAGXFFZ addr=172.31.7.66:40461 + ts=2023-08-15T01:40:08.840Z caller=nflog.go:275 level=debug component=nflog msg="notification log snapshot file doesn't exist" err="open /tmp/opni-test-2132242568/alertmanager_data/a450ba5c-4085-4ee7-98c2-15fca48f4d02/nflog: no such file or directory" + ts=2023-08-15T01:40:08.840Z caller=silence.go:353 level=debug component=silences msg="silences snapshot file doesn't exist" err="open /tmp/opni-test-2132242568/alertmanager_data/a450ba5c-4085-4ee7-98c2-15fca48f4d02/silences: no such file or directory" + ts=2023-08-15T01:40:08.840Z caller=cluster.go:266 level=debug component=cluster msg="joined cluster" peers=0 + ts=2023-08-15T01:40:08.840Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + ts=2023-08-15T01:40:08.872Z caller=alertmanager_main.go:413 level=debug externalURL=http://ip-172-31-7-66.us-east-2.compute.internal:43743 + ts=2023-08-15T01:40:08.873Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/webhook-4e35a247-5d11-489e-83c8-40fa597e3e97/config-qVtPRf6oS9VNSia69hynjA + ts=2023-08-15T01:40:08.873Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/webhook-4e35a247-5d11-489e-83c8-40fa597e3e97/config-qVtPRf6oS9VNSia69hynjA + ts=2023-08-15T01:40:08.875Z caller=alertmanager_main.go:543 level=debug routePrefix=/ + ts=2023-08-15T01:40:08.876Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:43743 + ts=2023-08-15T01:40:08.876Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:43743 + 2023-08-15T01:40:09Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:09Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:09Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:09Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:09Z INFO test.env AlertManager started {"address": "http://localhost:43743", "opni-address": "http://localhost:824688928536"} + STEP: sending alerts to each condition in the router @ 08/15/23 01:40:09.742 + ts=2023-08-15T01:40:09.743Z caller=dispatch.go:163 level=debug component=dispatcher msg="Received alert" alert=[f5800cc][active] + ts=2023-08-15T01:40:09.744Z caller=dispatch.go:163 level=debug component=dispatcher msg="Received alert" alert=[843c13a][active] + ts=2023-08-15T01:40:09.745Z caller=dispatch.go:163 level=debug component=dispatcher msg="Received alert" alert=[b92943f][active] + STEP: getting the AlertManager state @ 08/15/23 01:40:09.745 + 2023-08-15T01:40:10Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:10Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:10Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:10Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + ts=2023-08-15T01:40:10.744Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test!=\"\"}:{opni_uuid=\"0492e20c-7f62-45e6-a1df-458561ea691e\", test=\"0492e20c-7f62-45e6-a1df-458561ea691e\"}" msg=flushing alerts=[[f5800cc][active]] + ts=2023-08-15T01:40:10.745Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test!=\"\"}:{opni_uuid=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\", test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}" msg=flushing alerts=[[843c13a][active]] + ts=2023-08-15T01:40:10.745Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}:{opni_uuid=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\", test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}" msg=flushing alerts=[[843c13a][active]] + ts=2023-08-15T01:40:10.745Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test!=\"\"}:{opni_uuid=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\", test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}" msg=flushing alerts=[[b92943f][active]] + ts=2023-08-15T01:40:10.745Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}:{opni_uuid=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\", test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}" msg=flushing alerts=[[b92943f][active]] + STEP: getting the AlertManager state @ 08/15/23 01:40:10.748 + ts=2023-08-15T01:40:10.841Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000716812s + 2023-08-15T01:40:11Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:11Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:11Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:11Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:11.752 + 2023-08-15T01:40:12Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:12Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:12Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:12Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:12.756 + ts=2023-08-15T01:40:12.842Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=4.001435371s + 2023-08-15T01:40:13Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:13Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:13Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:13Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:13.759 + 2023-08-15T01:40:14Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:14Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:14Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:14Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:14.762 + ts=2023-08-15T01:40:14.843Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=6.002066983s + 2023-08-15T01:40:15Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:15Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:15Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:15Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:15.766 + 2023-08-15T01:40:16Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:16Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:16Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:16Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:16.77 + ts=2023-08-15T01:40:16.843Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=8.00277936s + 2023-08-15T01:40:17Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:17Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:17Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:17Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:17.774 + 2023-08-15T01:40:18Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:18Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:18Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:18Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:18.778 + ts=2023-08-15T01:40:18.844Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003654663s + ts=2023-08-15T01:40:18.846Z caller=notify.go:751 level=debug component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:18.847Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__a135cbb4-f039-4f29-9153-dfe0ae62dd32 integration=webhook[1] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:18.847Z caller=notify.go:751 level=debug component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:18.847Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__22e67d2a-f6c0-461e-8a14-ee6be7004cfd integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:18.848Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}:{opni_uuid=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\", test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}" msg=flushing alerts=[[843c13a][active]] + ts=2023-08-15T01:40:18.848Z caller=notify.go:751 level=debug component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:18.848Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__a135cbb4-f039-4f29-9153-dfe0ae62dd32 integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:18.848Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}:{opni_uuid=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\", test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}" msg=flushing alerts=[[b92943f][active]] + 2023-08-15T01:40:19Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:19Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:19Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:19Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:19.781 + STEP: updating an endpoint to another endpoint @ 08/15/23 01:40:19.784 + STEP: send an an alert to each specs @ 08/15/23 01:40:19.785 + STEP: expecting to successfully build a config @ 08/15/23 01:40:19.785 + STEP: checking that the config passes the marshalling validation @ 08/15/23 01:40:19.785 + STEP: checking that the config passes the unmarshalling validation @ 08/15/23 01:40:19.789 + STEP: writing the config file to a temporary directory @ 08/15/23 01:40:19.795 + STEP: Verifying that the config can be loaded by alertmanager @ 08/15/23 01:40:19.795 + 2023-08-15T01:40:19Z INFO test.env Waiting for alertmanager to start... + ts=2023-08-15T01:40:19.895Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:40:19.895Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + 2023-08-15T01:40:19Z INFO plugin.opni.alerting extensions/embedded_server.go:88 starting opni embedded server {"addr": ":45231"} + ts=2023-08-15T01:40:19.896Z caller=cluster.go:173 level=debug component=cluster msg="resolved peers to following addresses" peers= + ts=2023-08-15T01:40:19.898Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=37031 + ts=2023-08-15T01:40:19.899Z caller=delegate.go:236 level=debug component=cluster received=NotifyJoin node=01H7VDKPSTYQSVV8XAX0H1RNMS addr=172.31.7.66:37031 + ts=2023-08-15T01:40:19.899Z caller=nflog.go:275 level=debug component=nflog msg="notification log snapshot file doesn't exist" err="open /tmp/opni-test-2132242568/alertmanager_data/66988909-cc6e-44b7-8113-9a61c7ec8ca5/nflog: no such file or directory" + ts=2023-08-15T01:40:19.899Z caller=silence.go:353 level=debug component=silences msg="silences snapshot file doesn't exist" err="open /tmp/opni-test-2132242568/alertmanager_data/66988909-cc6e-44b7-8113-9a61c7ec8ca5/silences: no such file or directory" + ts=2023-08-15T01:40:19.899Z caller=cluster.go:266 level=debug component=cluster msg="joined cluster" peers=0 + ts=2023-08-15T01:40:19.899Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + ts=2023-08-15T01:40:19.929Z caller=alertmanager_main.go:413 level=debug externalURL=http://ip-172-31-7-66.us-east-2.compute.internal:35517 + ts=2023-08-15T01:40:19.929Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/webhook-4e35a247-5d11-489e-83c8-40fa597e3e97/config-9Z5Qi5BaPMc6fB5LtWxom3 + ts=2023-08-15T01:40:19.930Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/webhook-4e35a247-5d11-489e-83c8-40fa597e3e97/config-9Z5Qi5BaPMc6fB5LtWxom3 + ts=2023-08-15T01:40:19.932Z caller=alertmanager_main.go:543 level=debug routePrefix=/ + ts=2023-08-15T01:40:19.933Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:35517 + ts=2023-08-15T01:40:19.933Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:35517 + 2023-08-15T01:40:20Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:20Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:20Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:20Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:20Z INFO test.env AlertManager started {"address": "http://localhost:35517", "opni-address": "http://localhost:824693595272"} + STEP: sending alerts to each condition in the router @ 08/15/23 01:40:20.799 + ts=2023-08-15T01:40:20.801Z caller=dispatch.go:163 level=debug component=dispatcher msg="Received alert" alert=[f5800cc][active] + ts=2023-08-15T01:40:20.805Z caller=dispatch.go:163 level=debug component=dispatcher msg="Received alert" alert=[843c13a][active] + ts=2023-08-15T01:40:20.806Z caller=dispatch.go:163 level=debug component=dispatcher msg="Received alert" alert=[b92943f][active] + STEP: getting the AlertManager state @ 08/15/23 01:40:20.806 + 2023-08-15T01:40:21Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:21Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:21Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:21Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + ts=2023-08-15T01:40:21.802Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test!=\"\"}:{opni_uuid=\"0492e20c-7f62-45e6-a1df-458561ea691e\", test=\"0492e20c-7f62-45e6-a1df-458561ea691e\"}" msg=flushing alerts=[[f5800cc][active]] + ts=2023-08-15T01:40:21.806Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test!=\"\"}:{opni_uuid=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\", test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}" msg=flushing alerts=[[b92943f][active]] + ts=2023-08-15T01:40:21.806Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test!=\"\"}:{opni_uuid=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\", test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}" msg=flushing alerts=[[843c13a][active]] + ts=2023-08-15T01:40:21.806Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}:{opni_uuid=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\", test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}" msg=flushing alerts=[[843c13a][active]] + ts=2023-08-15T01:40:21.806Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}:{opni_uuid=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\", test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}" msg=flushing alerts=[[b92943f][active]] + STEP: getting the AlertManager state @ 08/15/23 01:40:21.809 + ts=2023-08-15T01:40:21.900Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000906662s + 2023-08-15T01:40:22Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:22Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:22Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:22Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:22.812 + 2023-08-15T01:40:23Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:23Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:23Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:23Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:23.816 + ts=2023-08-15T01:40:23.900Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=4.001237221s + 2023-08-15T01:40:24Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:24Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:24Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:24Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:24.82 + 2023-08-15T01:40:25Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:25Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:25Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:25Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:25.823 + ts=2023-08-15T01:40:25.901Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=6.001417472s + 2023-08-15T01:40:26Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:26Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:26Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:26Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:26.827 + 2023-08-15T01:40:27Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:27Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:27Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:27Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:27.831 + ts=2023-08-15T01:40:27.901Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=8.002091736s + 2023-08-15T01:40:28Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:28Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:28Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:28Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:28.834 + 2023-08-15T01:40:29Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:29Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:29Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:29Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:29.838 + ts=2023-08-15T01:40:29.902Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002835787s + ts=2023-08-15T01:40:29.905Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__22e67d2a-f6c0-461e-8a14-ee6be7004cfd integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:29.905Z caller=notify.go:751 level=debug component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:29.905Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__a135cbb4-f039-4f29-9153-dfe0ae62dd32 integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:29.905Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__a135cbb4-f039-4f29-9153-dfe0ae62dd32 integration=webhook[1] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:29.905Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}:{opni_uuid=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\", test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}" msg=flushing alerts=[[843c13a][active]] + ts=2023-08-15T01:40:29.905Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}:{opni_uuid=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\", test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}" msg=flushing alerts=[[b92943f][active]] + ts=2023-08-15T01:40:29.906Z caller=notify.go:751 level=debug component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:29.906Z caller=notify.go:751 level=debug component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify success" attempts=1 + 2023-08-15T01:40:30Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:30Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:30Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:30Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:30.841 + STEP: dynamic-alert-routing step: expecting that the router can build the config @ 08/15/23 01:40:30.844 + STEP: dynamic-alert-routing step: expecting that the formed alertmanager config is correct @ 08/15/23 01:40:30.844 + STEP: checking that the config passes the marshalling validation @ 08/15/23 01:40:30.845 + STEP: checking that the config passes the unmarshalling validation @ 08/15/23 01:40:30.849 + STEP: writing the config file to a temporary directory @ 08/15/23 01:40:30.854 + STEP: Verifying that the config can be loaded by alertmanager @ 08/15/23 01:40:30.854 + 2023-08-15T01:40:30Z INFO test.env Waiting for alertmanager to start... + ts=2023-08-15T01:40:30.954Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:40:30.954Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + ts=2023-08-15T01:40:30.954Z caller=cluster.go:173 level=debug component=cluster msg="resolved peers to following addresses" peers= + 2023-08-15T01:40:30Z INFO plugin.opni.alerting extensions/embedded_server.go:88 starting opni embedded server {"addr": ":33703"} + ts=2023-08-15T01:40:30.957Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=42007 + ts=2023-08-15T01:40:30.957Z caller=delegate.go:236 level=debug component=cluster received=NotifyJoin node=01H7VDM1KDKHQSMB0F8575GNBZ addr=172.31.7.66:42007 + ts=2023-08-15T01:40:30.958Z caller=nflog.go:275 level=debug component=nflog msg="notification log snapshot file doesn't exist" err="open /tmp/opni-test-2132242568/alertmanager_data/65d13bae-e61e-460f-854f-e03bfebcfaf1/nflog: no such file or directory" + ts=2023-08-15T01:40:30.958Z caller=silence.go:353 level=debug component=silences msg="silences snapshot file doesn't exist" err="open /tmp/opni-test-2132242568/alertmanager_data/65d13bae-e61e-460f-854f-e03bfebcfaf1/silences: no such file or directory" + ts=2023-08-15T01:40:30.958Z caller=cluster.go:266 level=debug component=cluster msg="joined cluster" peers=0 + ts=2023-08-15T01:40:30.958Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + ts=2023-08-15T01:40:30.989Z caller=alertmanager_main.go:413 level=debug externalURL=http://ip-172-31-7-66.us-east-2.compute.internal:36625 + ts=2023-08-15T01:40:30.989Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-d641c788-8419-4aa8-8c99-5c79ace97dc4/config-U9AJ2KpRYwn2padF6REDy4 + ts=2023-08-15T01:40:30.990Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-d641c788-8419-4aa8-8c99-5c79ace97dc4/config-U9AJ2KpRYwn2padF6REDy4 + ts=2023-08-15T01:40:30.992Z caller=alertmanager_main.go:543 level=debug routePrefix=/ + ts=2023-08-15T01:40:30.993Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:36625 + ts=2023-08-15T01:40:30.993Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:36625 + 2023-08-15T01:40:31Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:31Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:31Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:31Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:31Z INFO test.env AlertManager started {"address": "http://localhost:36625", "opni-address": "http://localhost:824705117080"} +• [34.192 seconds] +------------------------------ +metrics and alerting when When we use alerting on metrics should setup alertig & metrics clusters [integration] +/home/ec2-user/opni/test/plugins/alerting/metrics_test.go:37 + 2023-08-15T01:40:31Z INFO test.env Starting test environment + 2023-08-15T01:40:31Z INFO test.env Copying 7 files from embedded testdata/cortex to /tmp/opni-test-1343125945/cortex + 2023-08-15T01:40:31Z INFO test.env Waiting for jetstream to start... + [231568] 2023/08/15 01:40:31.870009 [INF] Starting nats-server + [231568] 2023/08/15 01:40:31.870058 [INF] Version: 2.9.21 + [231568] 2023/08/15 01:40:31.870062 [INF] Git: [b2e7725] + [231568] 2023/08/15 01:40:31.870066 [INF] Name: NBXVAIKYEU7FBTAO2673LMATBW2URPRH6FUVPMUF7F5QVLDRH27IZVNC + [231568] 2023/08/15 01:40:31.870072 [INF] Node: GU4SqiYQ + [231568] 2023/08/15 01:40:31.870076 [INF] ID: NBXVAIKYEU7FBTAO2673LMATBW2URPRH6FUVPMUF7F5QVLDRH27IZVNC + [231568] 2023/08/15 01:40:31.870098 [INF] Using configuration file: /tmp/opni-test-1343125945/jetstream/jetstream.conf + [231568] 2023/08/15 01:40:31.870791 [INF] Starting JetStream + [231568] 2023/08/15 01:40:31.870951 [INF] _ ___ _____ ___ _____ ___ ___ _ __ __ + [231568] 2023/08/15 01:40:31.871030 [INF] _ | | __|_ _/ __|_ _| _ \ __| /_\ | \/ | + [231568] 2023/08/15 01:40:31.871077 [INF] | || | _| | | \__ \ | | | / _| / _ \| |\/| | + [231568] 2023/08/15 01:40:31.871092 [INF] \__/|___| |_| |___/ |_| |_|_\___/_/ \_\_| |_| + [231568] 2023/08/15 01:40:31.871096 [INF] + [231568] 2023/08/15 01:40:31.871099 [INF] https://docs.nats.io/jetstream + [231568] 2023/08/15 01:40:31.871102 [INF] + [231568] 2023/08/15 01:40:31.871105 [INF] ---------------- JETSTREAM ---------------- + [231568] 2023/08/15 01:40:31.871112 [INF] Max Memory: 11.71 GB + [231568] 2023/08/15 01:40:31.871117 [INF] Max Storage: 5.29 GB + [231568] 2023/08/15 01:40:31.871120 [INF] Store Directory: "/tmp/opni-test-1343125945/jetstream/data/jetstream" + [231568] 2023/08/15 01:40:31.871123 [INF] ------------------------------------------- + [231568] 2023/08/15 01:40:31.871577 [INF] Listening for client connections on 0.0.0.0:32799 + [231568] 2023/08/15 01:40:31.871952 [INF] Server is ready + 2023-08-15T01:40:31Z INFO test.env Jetstream started + 2023-08-15T01:40:31Z DEBUG keyring loaded ephemeral key {"path": "/tmp/opni-test-1343125945/keyring/local-agent.json", "usage": "auth", "labels": {"opni.io/session-attribute":"local"}} + 2023-08-15T01:40:31Z DEBUG gateway found 0 plugins + 2023-08-15T01:40:31Z DEBUG gateway loaded plugin manifest {"plugins": 0} + 2023-08-15T01:40:31Z INFO gateway.cache compressing and archiving plugins... + 2023-08-15T01:40:31Z DEBUG gateway.cache added 0 new plugins to cache + 2023-08-15T01:40:31Z INFO gateway running plugin cache gc + 2023-08-15T01:40:31Z INFO gateway.update-server registering update handler for strategy "binary" + 2023-08-15T01:40:31Z INFO gateway.update-server registering update handler for strategy "kubernetes" + 2023-08-15T01:40:31Z INFO gateway.update-server registering update handler for strategy "noop" + 2023-08-15T01:40:31Z DEBUG gateway.grpc registering service {"service": "control.HealthListener"} + 2023-08-15T01:40:31Z DEBUG gateway.grpc registering internal service {"service": "stream.Delegate"} + 2023-08-15T01:40:31Z DEBUG gateway.grpc registering service {"service": "core.Pinger"} + 2023-08-15T01:40:31Z INFO test.env Loading gateway plugins... + 2023-08-15T01:40:31Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:31Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.System", "opni.apiextensions.HTTPAPIExtension", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.ManagementAPIExtension", "opni.backends.Capability", "opni.Metrics"]} + 2023-08-15T01:40:31Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.Metrics", "opni.backends.Capability", "opni.apiextensions.StreamAPIExtension", "opni.System", "opni.apiextensions.HTTPAPIExtension", "opni.apiextensions.ManagementAPIExtension"]} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Metrics"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_metrics"} + 2023-08-15T01:40:31Z DEBUG plugin.metrics/pkg/gateway.stream internal gateway stream connected + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:31Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/alerting", "capability": "alerting"} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_alerting"} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Metrics"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:31Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/metrics", "capability": "metrics"} + 2023-08-15T01:40:31Z INFO mgmt got extension descriptor for service cortexadmin.CortexAdmin + 2023-08-15T01:40:31Z DEBUG gateway.grpc plugin rejected internal stream connection {"plugin": "plugin_alerting"} + 2023-08-15T01:40:31Z DEBUG plugin.metrics/pkg/gateway.stream calling client handler + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.System"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.System"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/trace", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/profile", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/block", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/heap", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/ready", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/healthy", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z INFO mgmt loading service {"name": "cortexadmin.CortexAdmin"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/AllUserStats"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/WriteMetrics"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/Query"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/QueryRange"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetRule"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricMetadata"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ListRules"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/LoadRules"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/DeleteRule"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/FlushBlocks"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetSeriesMetrics"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricLabelSets"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexStatus"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexConfig"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ExtractRawSeries"} + 2023-08-15T01:40:31Z INFO mgmt loading http rules {"name": "cortexadmin.CortexAdmin", "rules": 17} + 2023-08-15T01:40:31Z DEBUG mgmt rule descriptors {"name": "cortexadmin.CortexAdmin", "rules": [{"http":{"Pattern":{"Get":"/all_user_stats"}},"method":{"name":"AllUserStats","input_type":".google.protobuf.Empty","output_type":".cortexadmin.UserIDStatsList","options":{}}},{"http":{"Pattern":{"Post":"/write_metrics"},"body":"*"},"method":{"name":"WriteMetrics","input_type":".cortexadmin.WriteRequest","output_type":".cortexadmin.WriteResponse","options":{}}},{"http":{"Pattern":{"Get":"/query"},"additional_bindings":[{"Pattern":{"Post":"/query"},"body":"*"}]},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query"},"body":"*"},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/query_range"},"additional_bindings":[{"Pattern":{"Post":"/query_range"},"body":"*"}]},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query_range"},"body":"*"},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/rules/{namespace}/{groupName}"}},"method":{"name":"GetRule","input_type":".cortexadmin.GetRuleRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/metadata"}},"method":{"name":"GetMetricMetadata","input_type":".cortexadmin.MetricMetadataRequest","output_type":".cortexadmin.MetricMetadata","options":{}}},{"http":{"Pattern":{"Get":"/rules"}},"method":{"name":"ListRules","input_type":".cortexadmin.ListRulesRequest","output_type":".cortexadmin.ListRulesResponse","options":{}}},{"http":{"Pattern":{"Post":"/rules"}},"method":{"name":"LoadRules","input_type":".cortexadmin.LoadRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/rules/{groupName}"}},"method":{"name":"DeleteRule","input_type":".cortexadmin.DeleteRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/flush_blocks"}},"method":{"name":"FlushBlocks","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/series/metadata"}},"method":{"name":"GetSeriesMetrics","input_type":".cortexadmin.SeriesRequest","output_type":".cortexadmin.SeriesInfoList","options":{}}},{"http":{"Pattern":{"Get":"/series/labels"}},"method":{"name":"GetMetricLabelSets","input_type":".cortexadmin.LabelRequest","output_type":".cortexadmin.MetricLabels","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetCortexStatus","input_type":".google.protobuf.Empty","output_type":".cortexadmin.CortexStatus","options":{}}},{"http":{"Pattern":{"Get":"/config"}},"method":{"name":"GetCortexConfig","input_type":".cortexadmin.ConfigRequest","output_type":".cortexadmin.ConfigResponse","options":{}}},{"http":{"Pattern":{"Get":"/series/raw"}},"method":{"name":"ExtractRawSeries","input_type":".cortexadmin.MatcherRequest","output_type":".cortexadmin.QueryResponse","options":{}}}]} + 2023-08-15T01:40:31Z INFO mgmt got extension descriptor for service cortexops.CortexOps + 2023-08-15T01:40:31Z INFO mgmt got extension descriptor for service alerting.AlertConditions + 2023-08-15T01:40:31Z INFO mgmt loading service {"name": "cortexops.CortexOps"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterConfiguration"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexops.CortexOps/ConfigureCluster"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterStatus"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexops.CortexOps/UninstallCluster"} + 2023-08-15T01:40:31Z INFO mgmt loading http rules {"name": "cortexops.CortexOps", "rules": 4} + 2023-08-15T01:40:31Z DEBUG mgmt rule descriptors {"name": "cortexops.CortexOps", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".cortexops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".cortexops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".cortexops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:40:31Z INFO mgmt got extension descriptor for service remoteread.RemoteReadGateway + 2023-08-15T01:40:31Z INFO mgmt loading service {"name": "remoteread.RemoteReadGateway"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/AddTarget"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/EditTarget"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/RemoveTarget"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/ListTargets"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Start"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Stop"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/GetTargetStatus"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Discover"} + 2023-08-15T01:40:31Z INFO mgmt loading http rules {"name": "remoteread.RemoteReadGateway", "rules": 8} + 2023-08-15T01:40:31Z DEBUG mgmt rule descriptors {"name": "remoteread.RemoteReadGateway", "rules": [{"http":{"Pattern":{"Post":"/targets"},"body":"*"},"method":{"name":"AddTarget","input_type":".remoteread.TargetAddRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Put":"/targets"},"body":"*"},"method":{"name":"EditTarget","input_type":".remoteread.TargetEditRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/targets/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"RemoveTarget","input_type":".remoteread.TargetRemoveRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/targets"}},"method":{"name":"ListTargets","input_type":".remoteread.TargetListRequest","output_type":".remoteread.TargetList","options":{}}},{"http":{"Pattern":{"Post":"/targets/start/{target.meta.clusterId}/{target.meta.name}"},"body":"*"},"method":{"name":"Start","input_type":".remoteread.StartReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/targets/stop/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"Stop","input_type":".remoteread.StopReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status"},"body":"*"},"method":{"name":"GetTargetStatus","input_type":".remoteread.TargetStatusRequest","output_type":".remoteread.TargetStatus","options":{}}},{"http":{"Pattern":{"Post":"/edit"},"body":"*"},"method":{"name":"Discover","input_type":".remoteread.DiscoveryRequest","output_type":".remoteread.DiscoveryResponse","options":{}}}]} + 2023-08-15T01:40:31Z INFO mgmt got extension descriptor for service node.metrics.NodeConfiguration + 2023-08-15T01:40:32Z INFO mgmt loading service {"name": "node.metrics.NodeConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading http rules {"name": "node.metrics.NodeConfiguration", "rules": 4} + 2023-08-15T01:40:32Z DEBUG mgmt rule descriptors {"name": "node.metrics.NodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".node.metrics.MetricsCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".node.metrics.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:40:32Z INFO plugin.alerting using cluster driver {"driver": "test-environment"} + 2023-08-15T01:40:32Z INFO plugin.opni.alerting starting opni embedded server {"addr": "127.0.0.1:44103"} + 2023-08-15T01:40:32Z INFO plugin.metrics.cortex-http configuring http api server + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /ready", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PUT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PATCH /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "HEAD /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "DELETE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "DELETE /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "TRACE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z INFO mgmt loading service {"name": "alerting.AlertConditions"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionGroups"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CreateAlertCondition"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/GetAlertCondition"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditions"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/UpdateAlertCondition"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionChoices"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeleteAlertCondition"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/AlertConditionStatus"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionsWithStatus"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CloneTo"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ActivateSilence"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeactivateSilence"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/Timeline"} + 2023-08-15T01:40:32Z INFO mgmt loading http rules {"name": "alerting.AlertConditions", "rules": 13} + 2023-08-15T01:40:32Z DEBUG mgmt rule descriptors {"name": "alerting.AlertConditions", "rules": [{"http":{"Pattern":{"Get":"/groups"}},"method":{"name":"ListAlertConditionGroups","input_type":".google.protobuf.Empty","output_type":".core.ReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertCondition","input_type":".alerting.AlertCondition","output_type":".alerting.ConditionReference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertCondition","input_type":".alerting.ConditionReference","output_type":".alerting.AlertCondition","options":{}}},{"http":{"Pattern":{"Post":"/list"},"body":"*"},"method":{"name":"ListAlertConditions","input_type":".alerting.ListAlertConditionRequest","output_type":".alerting.AlertConditionList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertCondition","input_type":".alerting.UpdateAlertConditionRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/choices"}},"method":{"name":"ListAlertConditionChoices","input_type":".alerting.AlertDetailChoicesRequest","output_type":".alerting.ListAlertTypeDetails","options":{}}},{"http":{"Pattern":{"Delete":"/configure"}},"method":{"name":"DeleteAlertCondition","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status/{id}"}},"method":{"name":"AlertConditionStatus","input_type":".alerting.ConditionReference","output_type":".alerting.AlertStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/list/withStatus"},"body":"*"},"method":{"name":"ListAlertConditionsWithStatus","input_type":".alerting.ListStatusRequest","output_type":".alerting.ListStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/clone"},"body":"*"},"method":{"name":"CloneTo","input_type":".alerting.CloneToRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/silences"},"body":"*"},"method":{"name":"ActivateSilence","input_type":".alerting.SilenceRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/silences"}},"method":{"name":"DeactivateSilence","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/timeline"}},"method":{"name":"Timeline","input_type":".alerting.TimelineRequest","output_type":".alerting.TimelineResponse","options":{}}}]} + 2023-08-15T01:40:32Z INFO mgmt got extension descriptor for service alerting.AlertEndpoints + 2023-08-15T01:40:32Z INFO mgmt loading service {"name": "alerting.AlertEndpoints"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/CreateAlertEndpoint"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/GetAlertEndpoint"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ToggleNotifications"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ListAlertEndpoints"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/UpdateAlertEndpoint"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/DeleteAlertEndpoint"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/TestAlertEndpoint"} + 2023-08-15T01:40:32Z INFO mgmt loading http rules {"name": "alerting.AlertEndpoints", "rules": 7} + 2023-08-15T01:40:32Z DEBUG mgmt rule descriptors {"name": "alerting.AlertEndpoints", "rules": [{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertEndpoint","input_type":".alerting.AlertEndpoint","output_type":".core.Reference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertEndpoint","input_type":".core.Reference","output_type":".alerting.AlertEndpoint","options":{}}},{"http":{"Pattern":{"Post":"/toggleNotifications"},"body":"*"},"method":{"name":"ToggleNotifications","input_type":".alerting.ToggleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/list"}},"method":{"name":"ListAlertEndpoints","input_type":".alerting.ListAlertEndpointsRequest","output_type":".alerting.AlertEndpointList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertEndpoint","input_type":".alerting.UpdateAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/delete"},"body":"*"},"method":{"name":"DeleteAlertEndpoint","input_type":".alerting.DeleteAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/test"},"body":"*"},"method":{"name":"TestAlertEndpoint","input_type":".alerting.TestAlertEndpointRequest","output_type":".alerting.TestAlertEndpointResponse","options":{}}}]} + 2023-08-15T01:40:32Z INFO mgmt got extension descriptor for service alerting.AlertNotifications + 2023-08-15T01:40:32Z INFO mgmt loading service {"name": "alerting.AlertNotifications"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/TriggerAlerts"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ResolveAlerts"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/PushNotification"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListNotifications"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListAlarmMessages"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListRoutingRelationships"} + 2023-08-15T01:40:32Z INFO mgmt loading http rules {"name": "alerting.AlertNotifications", "rules": 4} + 2023-08-15T01:40:32Z DEBUG mgmt rule descriptors {"name": "alerting.AlertNotifications", "rules": [{"http":{"Pattern":{"Post":"/trigger"},"body":"*"},"method":{"name":"TriggerAlerts","input_type":".alerting.TriggerAlertsRequest","output_type":".alerting.TriggerAlertsResponse","options":{}}},{"http":{"Pattern":{"Post":"/push"},"body":"*"},"method":{"name":"PushNotification","input_type":".alerting.Notification","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/notifications/list"},"body":"*"},"method":{"name":"ListNotifications","input_type":".alerting.ListNotificationRequest","output_type":".alerting.ListMessageResponse","options":{}}},{"http":{"Pattern":{"Post":"/alarms/list"},"body":"*"},"method":{"name":"ListAlarmMessages","input_type":".alerting.ListAlarmMessageRequest","output_type":".alerting.ListMessageResponse","options":{}}}]} + 2023-08-15T01:40:32Z INFO mgmt got extension descriptor for service alerting.ops.AlertingAdmin + 2023-08-15T01:40:32Z INFO mgmt loading service {"name": "alerting.ops.AlertingAdmin"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/ConfigureCluster"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterStatus"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/InstallCluster"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/UninstallCluster"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/Info"} + 2023-08-15T01:40:32Z INFO mgmt loading http rules {"name": "alerting.ops.AlertingAdmin", "rules": 6} + 2023-08-15T01:40:32Z DEBUG mgmt rule descriptors {"name": "alerting.ops.AlertingAdmin", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".alerting.ops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".alerting.ops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/install"}},"method":{"name":"InstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".alerting.ops.UninstallRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/info"}},"method":{"name":"Info","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ComponentInfo","options":{}}}]} + 2023-08-15T01:40:32Z INFO mgmt got extension descriptor for service alerting.ops.ConfigReconciler + 2023-08-15T01:40:32Z INFO mgmt loading service {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.ops.ConfigReconciler/SyncConfig"} + 2023-08-15T01:40:32Z INFO mgmt service has no http rules {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:40:32Z INFO mgmt got extension descriptor for service alerting.node.AlertingNodeConfiguration + 2023-08-15T01:40:32Z INFO mgmt loading service {"name": "alerting.node.AlertingNodeConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading http rules {"name": "alerting.node.AlertingNodeConfiguration", "rules": 4} + 2023-08-15T01:40:32Z DEBUG mgmt rule descriptors {"name": "alerting.node.AlertingNodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".alerting.node.AlertingCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".alerting.node.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:40:32Z INFO mgmt got extension descriptor for service alerting.node.NodeAlertingCapability + 2023-08-15T01:40:32Z INFO mgmt loading service {"name": "alerting.node.NodeAlertingCapability"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.node.NodeAlertingCapability/Sync"} + 2023-08-15T01:40:32Z INFO mgmt loading http rules {"name": "alerting.node.NodeAlertingCapability", "rules": 1} + 2023-08-15T01:40:32Z DEBUG mgmt rule descriptors {"name": "alerting.node.NodeAlertingCapability", "rules": [{"http":{"Pattern":{"Post":"/v1/node/sync"},"body":"*"},"method":{"name":"Sync","input_type":".alerting.node.AlertingCapabilityConfig","output_type":".alerting.node.SyncResponse","options":{}}}]} + 2023-08-15T01:40:32Z INFO test.env Waiting for gateway to start... + 2023-08-15T01:40:32Z INFO mgmt management HTTP server starting {"address": "localhost:45639"} + 2023-08-15T01:40:32Z INFO gateway.http gateway HTTP server starting {"api": "127.0.0.1:45615", "metrics": "127.0.0.1:36647"} + 2023-08-15T01:40:32Z INFO test.env loaded 12 plugins + 2023-08-15T01:40:32Z INFO gateway.grpc gateway gRPC server starting {"address": "127.0.0.1:36691"} + 2023-08-15T01:40:32Z INFO mgmt management gRPC server starting {"address": "127.0.0.1:46283"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/tokens"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/tokens/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/watch"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/clusters/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/certs"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}/health"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/health/watch"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/clusters/{cluster.id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/roles"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/roles"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/roles/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/rolebindings"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/rolebindings"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/subjectaccess"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/apiextensions"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/config"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/config"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/capabilities"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/capabilities/{name}/installer"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/install"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/uninstall"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/status"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/status"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/cancel"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/dashboard/settings"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/dashboard/settings"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/all_user_stats"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/write_metrics"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules/{namespace}/{groupName}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/metadata"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/rules"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/CortexAdmin/rules/{groupName}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/flush_blocks"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/metadata"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/labels"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/status"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/config"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/raw"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/configuration"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/configure"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/status"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/uninstall"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:40:32Z INFO test.env Gateway started + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/RemoteReadGateway/targets/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/start/{target.meta.clusterId}/{target.meta.name}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/stop/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/status"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/edit"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config/{node.id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertConditions/groups"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/configure"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertConditions/configure"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/choices"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/configure"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/status/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/withStatus"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/clone"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/silences"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/silences"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/timeline"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/list/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/toggleNotifications"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertEndpoints/list"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/delete"} + 2023-08-15T01:40:32Z DEBUG plugin.alerting-test-cluster-driver Syncer start : alerting-server --syncer.alertmanager.config.file=/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml --syncer.listen.address=:38629 --syncer.alertmanager.address=http://127.0.0.1:44497 --syncer.gateway.join.address=:46283 syncer {"config-file": "/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/test"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/trigger"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/push"} + 2023-08-15T01:40:32Z INFO plugin.alerting-test-cluster-driver Starting AlertManager {"config-file": "/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml", "alertmanager-port": 44497, "opni-port": 40425} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/notifications/list"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/alarms/list"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/configuration"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/configure"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/status"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/install"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/uninstall"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/info"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config/{node.id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/NodeAlertingCapability/v1/node/sync"} + ts=2023-08-15T01:40:32.162Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:40:32.162Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + ts=2023-08-15T01:40:32.165Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=39079 + ts=2023-08-15T01:40:32.166Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + ts=2023-08-15T01:40:32.197Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml + ts=2023-08-15T01:40:32.199Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml + ts=2023-08-15T01:40:32.201Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:44497 + ts=2023-08-15T01:40:32.201Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:44497 + 2023-08-15T01:40:32Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:32Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:32Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:32Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:32Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:32Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:32Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:32Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + ts=2023-08-15T01:40:32.959Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000539108s + 2023-08-15T01:40:33Z INFO plugin.alerting-test-cluster-driver Starting AlertManager Syncer {"config-file": "/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml", "port": 38629} + 2023-08-15T01:40:33Z INFO plugin.alerting-test-cluster-driver AlertManager started {"config-file": "/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml", "address": "http://127.0.0.1:44497"} + 2023-08-15T01:40:33Z INFO plugin.alerting updating known peers : [{http://127.0.0.1:44497 http://127.0.0.1:44103}] + 2023-08-15T01:40:33Z INFO plugin.metrics.metrics-backend node sync requested {"cluster": "(all)", "capability": "metrics"} + 2023-08-15T01:40:33Z INFO test.env Waiting for cortex to start... + 2023-08-15T01:40:33Z INFO test starting agent {"id": "agent1", "address": "127.0.0.1:46699", "version": "v2"} + 2023-08-15T01:40:33Z DEBUG agent using log level: debug + 2023-08-15T01:40:33Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:40:33Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent1 + 2023-08-15T01:40:33Z INFO agent bootstrap completed successfully + 2023-08-15T01:40:33Z WARN jetstream disconnected from jetstream + 2023-08-15T01:40:33Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:40:33Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent1 is now active + 2023-08-15T01:40:33Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent1 is now active + 2023-08-15T01:40:33Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:40:33Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.Health", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.HTTPAPIExtension", "opni.backends.Capability"]} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:40:33Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:40:33Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:33Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:33Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:40:33Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:40:33Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:40:33Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:33Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:33Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:40:33Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:40:33Z INFO agent using unmanaged plugin loader + 2023-08-15T01:40:33Z INFO agent agent http server starting {"address": "127.0.0.1:46699"} + 2023-08-15T01:40:33Z INFO agent connecting to gateway... + 2023-08-15T01:40:33Z DEBUG gateway ratelimit: 50.000000 available + 2023-08-15T01:40:33Z DEBUG gateway.grpc handling new stream connection + 2023-08-15T01:40:33Z DEBUG gateway-client authenticated {"id": "agent1"} + 2023-08-15T01:40:33Z DEBUG commands/alerting.go:72 syncer gateway join address:46283 + 2023-08-15T01:40:33Z INFO plugin.alerting-syncer syncer/syncer_server.go:90 starting alerting syncer server as identity 6af4aa22-8ad6-4656-b187-103ece63a491 + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent1"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream stream connected + ts=2023-08-15T01:40:33.18017799Z caller=experimental.go:19 level=warn msg="experimental feature in use" feature=tenant-federation + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:108 acquired gateway alerting client + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:144 starting alerting syncer stream subscription + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:150 starting new stream subscription + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent1"} + 2023-08-15T01:40:33Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "6e33b99a-7400-45fd-8d69-2d0f7dddaf82"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:40:33Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "bd425fda-bda6-4fee-ad58-e7b42139b468"} + ts=2023-08-15T01:40:33.188315242Z caller=modules.go:331 level=warn msg="Worker address is empty in single binary mode. Attempting automatic worker configuration. If queries are unresponsive consider configuring the worker explicitly." address=127.0.0.1:37009 + 2023-08-15T01:40:33Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml"} + ts=2023-08-15T01:40:33.198Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml + ts=2023-08-15T01:40:33.199Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml"} + 2023-08-15T01:40:33Z INFO test.env Waiting for cortex to start... {"status": "503 Service Unavailable"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:40:33Z DEBUG gateway.sync agent connected {"id": "agent1"} + 2023-08-15T01:40:33Z INFO gateway.monitor received status update {"id": "agent1", "connected": true} + 2023-08-15T01:40:33Z DEBUG gateway.delegate agent connected {"id": "agent1"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:40:33Z INFO gateway.monitor received health update {"id": "agent1", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:40:33Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z INFO agent gateway connected + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:40:33Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:40:33Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:40:33Z"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent1", "time": "2023-08-15T01:40:33Z"} + 2023-08-15T01:40:33Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:33Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting rule sync is disabled {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:40:33Z INFO gateway.monitor received health update {"id": "agent1", "ready": true, "conditions": []} + 2023-08-15T01:40:33Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:33Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:40:33Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent1"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:40:33Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:33Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent1", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:33Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:33Z INFO test.env Cortex started {"httpAddress": "https://localhost:41355", "grpcAddress": "localhost:37009"} + 2023-08-15T01:40:33Z INFO test starting agent {"id": "agent2", "address": "127.0.0.1:40937", "version": "v2"} + 2023-08-15T01:40:33Z DEBUG agent using log level: debug + 2023-08-15T01:40:33Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:40:33Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent2 + 2023-08-15T01:40:33Z INFO agent bootstrap completed successfully + 2023-08-15T01:40:33Z WARN jetstream disconnected from jetstream + 2023-08-15T01:40:33Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent2 is now active + 2023-08-15T01:40:33Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:40:33Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent2 is now active + 2023-08-15T01:40:33Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.HTTPAPIExtension"]} + 2023-08-15T01:40:33Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:40:33Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:33Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:33Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:40:33Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:40:33Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:40:33Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:33Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:33Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:40:33Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:40:33Z INFO agent using unmanaged plugin loader + 2023-08-15T01:40:33Z INFO agent agent http server starting {"address": "127.0.0.1:40937"} + 2023-08-15T01:40:33Z INFO agent connecting to gateway... + 2023-08-15T01:40:33Z DEBUG gateway ratelimit: 50.000000 available + 2023-08-15T01:40:33Z DEBUG gateway-client authenticated {"id": "agent2"} + 2023-08-15T01:40:33Z DEBUG gateway.grpc handling new stream connection + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent2"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream stream connected + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent2"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:40:33Z INFO gateway.monitor received status update {"id": "agent2", "connected": true} + 2023-08-15T01:40:33Z DEBUG gateway.sync agent connected {"id": "agent2"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:40:33Z DEBUG gateway.delegate agent connected {"id": "agent2"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:40:33Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:40:33Z INFO agent gateway connected + 2023-08-15T01:40:33Z INFO gateway.monitor received health update {"id": "agent2", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:40:33Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent2", "time": "2023-08-15T01:40:33Z"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent2", "time": "2023-08-15T01:40:33Z"} + 2023-08-15T01:40:33Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:33Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:40:33Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:33Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent2"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:40:33Z DEBUG test.env nothing to do (already stopped) {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent2", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG test.env nothing to do (already stopped) {"node": "agent2", "driver": "otel"} + 2023-08-15T01:40:33Z INFO gateway.monitor received health update {"id": "agent2", "ready": true, "conditions": []} + 2023-08-15T01:40:33Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting rule sync is disabled {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:33Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:33Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:40:33Z INFO test starting agent {"id": "agent3", "address": "127.0.0.1:46631", "version": "v2"} + 2023-08-15T01:40:33Z DEBUG agent using log level: debug + 2023-08-15T01:40:33Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:40:33Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent3 + 2023-08-15T01:40:33Z INFO agent bootstrap completed successfully + 2023-08-15T01:40:33Z WARN jetstream disconnected from jetstream + 2023-08-15T01:40:33Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:40:33Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent3 is now active + 2023-08-15T01:40:33Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent3 is now active + 2023-08-15T01:40:33Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.HTTPAPIExtension"]} + 2023-08-15T01:40:33Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:40:33Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:33Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:33Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:40:33Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:40:33Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:40:33Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:33Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:33Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:40:33Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:40:33Z INFO agent using unmanaged plugin loader + 2023-08-15T01:40:33Z INFO agent agent http server starting {"address": "127.0.0.1:46631"} + 2023-08-15T01:40:33Z INFO agent connecting to gateway... + 2023-08-15T01:40:33Z DEBUG gateway ratelimit: 50.000000 available + 2023-08-15T01:40:33Z DEBUG gateway-client authenticated {"id": "agent3"} + 2023-08-15T01:40:33Z DEBUG gateway.grpc handling new stream connection + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent3"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream stream connected + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent3"} + 2023-08-15T01:40:33Z DEBUG gateway.sync agent connected {"id": "agent3"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:40:33Z INFO gateway.monitor received status update {"id": "agent3", "connected": true} + 2023-08-15T01:40:33Z DEBUG gateway.delegate agent connected {"id": "agent3"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:33Z INFO gateway.monitor received health update {"id": "agent3", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:40:33Z INFO agent gateway connected + 2023-08-15T01:40:33Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:40:33Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent3", "time": "2023-08-15T01:40:33Z"} + 2023-08-15T01:40:33Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:33Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:40:33Z INFO gateway.monitor received health update {"id": "agent3", "ready": true, "conditions": []} + 2023-08-15T01:40:33Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent3"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:40:33Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent3", "driver": "prometheus"} + 2023-08-15T01:40:33Z DEBUG test.env nothing to do (already stopped) {"node": "agent3", "driver": "prometheus"} + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent3", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG test.env nothing to do (already stopped) {"node": "agent3", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:40:33Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:40:33Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent3", "time": "2023-08-15T01:40:33Z"} + 2023-08-15T01:40:33Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:33Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent3"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting rule sync is disabled {"component": "rule-streamer", "nodeId": "agent3"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:33Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:33Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:33Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} +• [1.834 seconds] +------------------------------ +metrics and alerting when When we use alerting on metrics should create prometheus query alerts that map to endpoints [integration] +/home/ec2-user/opni/test/plugins/alerting/metrics_test.go:87 + STEP: creating webhook endpoints for receiving the prometheus alerting @ 08/15/23 01:40:33.698 + STEP: creating prometheus query alert conditions @ 08/15/23 01:40:33.713 + STEP: making sure when metrics aren't installed, the conditions are invalid @ 08/15/23 01:40:33.732 +• [0.036 seconds] +------------------------------ +metrics and alerting when When we use alerting on metrics the metrics -> alerting pipeline should be functional [integration] +/home/ec2-user/opni/test/plugins/alerting/metrics_test.go:180 + STEP: installing the metrics capabilities @ 08/15/23 01:40:33.733 + 2023-08-15T01:40:33Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent1", "capabilities": ["metrics"]} + 2023-08-15T01:40:33Z DEBUG plugin.alerting ignoring sync request due to capability filter 'alerting' {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics received sync request + 2023-08-15T01:40:33Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:40:33Z INFO plugin.metrics.metrics-backend node sync requested {"cluster": "agent1", "capability": "metrics"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent1", "time": "2023-08-15T01:40:33Z"} + 2023-08-15T01:40:33Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent2", "capabilities": ["metrics"]} + 2023-08-15T01:40:33Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:40:33Z DEBUG plugin.metrics received sync request + 2023-08-15T01:40:33Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:40:33Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent1"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics starting rule sync + 2023-08-15T01:40:33Z DEBUG plugin.metrics enabling http server + 2023-08-15T01:40:33Z DEBUG plugin.alerting ignoring sync request due to capability filter 'alerting' {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics configuring rule discovery + 2023-08-15T01:40:33Z DEBUG plugin.metrics rule discovery configured + 2023-08-15T01:40:33Z DEBUG plugin.metrics rule discovery notifier configured {"interval": "15m0s"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics starting rule group update notifier + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent1", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:33Z INFO test.env starting prometheus {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent2", "time": "2023-08-15T01:40:33Z"} + 2023-08-15T01:40:33Z INFO test.env Waiting for prometheus to start... + 2023-08-15T01:40:33Z INFO plugin.metrics.metrics-backend node sync requested {"cluster": "agent2", "capability": "metrics"} + 2023-08-15T01:40:33Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:40:33Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent3", "capabilities": ["metrics"]} + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent2", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG test.env nothing to do (already stopped) {"node": "agent2", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:40:33Z INFO test.env starting prometheus {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics received sync request + 2023-08-15T01:40:33Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:40:33Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent2"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics starting rule sync + 2023-08-15T01:40:33Z DEBUG plugin.metrics enabling http server + 2023-08-15T01:40:33Z INFO test.env Waiting for prometheus to start... + 2023-08-15T01:40:33Z DEBUG plugin.metrics configuring rule discovery + 2023-08-15T01:40:33Z DEBUG plugin.metrics rule discovery configured + 2023-08-15T01:40:33Z DEBUG plugin.metrics rule discovery notifier configured {"interval": "15m0s"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics starting rule group update notifier + 2023-08-15T01:40:33Z DEBUG plugin.alerting ignoring sync request due to capability filter 'alerting' {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z INFO plugin.metrics.metrics-backend node sync requested {"cluster": "agent3", "capability": "metrics"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent3", "time": "2023-08-15T01:40:33Z"} + STEP: verifying the conditions move to the firing state @ 08/15/23 01:40:33.761 + STEP: making sure when metrics aren't installed, the conditions are invalid @ 08/15/23 01:40:33.761 + STEP: verifying the webhook endpoints have received the message @ 08/15/23 01:40:33.761 + 2023-08-15T01:40:33Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:40:33Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent3"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics starting rule sync + 2023-08-15T01:40:33Z DEBUG plugin.metrics enabling http server + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent3", "driver": "prometheus"} + 2023-08-15T01:40:33Z INFO test.env starting prometheus {"node": "agent3", "driver": "prometheus"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics configuring rule discovery + 2023-08-15T01:40:33Z DEBUG plugin.metrics rule discovery configured + 2023-08-15T01:40:33Z DEBUG plugin.metrics rule discovery notifier configured {"interval": "15m0s"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics starting rule group update notifier + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent3", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG test.env nothing to do (already stopped) {"node": "agent3", "driver": "otel"} + 2023-08-15T01:40:33Z INFO test.env Waiting for prometheus to start... + 2023-08-15T01:40:33Z INFO test.env Prometheus started {"address": "http://localhost:34255", "dir": "/tmp/opni-test-1343125945/prometheus/agent2", "agentId": "agent2"} + 2023-08-15T01:40:33Z INFO test.env started prometheus {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:40:33Z INFO test.env Prometheus started {"address": "http://localhost:43691", "dir": "/tmp/opni-test-1343125945/prometheus/agent1", "agentId": "agent1"} + 2023-08-15T01:40:33Z INFO test.env started prometheus {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:33Z INFO test.env Prometheus started {"address": "http://localhost:36529", "dir": "/tmp/opni-test-1343125945/prometheus/agent3", "agentId": "agent3"} + 2023-08-15T01:40:33Z INFO test.env started prometheus {"node": "agent3", "driver": "prometheus"} + 2023-08-15T01:40:33Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:33Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting received condition update: name:"agent-disconnect" description:"Alert when the downstream agent disconnects from the opni upstream" labels:"agent-disconnect" labels:"opni" labels:"_default" severity:Critical alertType:{system:{clusterId:{id:"agent3"} timeout:{seconds:1}}} lastUpdated:{seconds:1692063633 nanos:521667536} id:"qqYBdqk7WPkARoRmrZQ6vX" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onSystemConditionCreate": "qqYBdqk7WPkARoRmrZQ6vX"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting received condition update: name:"agent-disconnect" description:"Alert when the downstream agent disconnects from the opni upstream" labels:"agent-disconnect" labels:"opni" labels:"_default" severity:Critical alertType:{system:{clusterId:{id:"agent1"} timeout:{seconds:1}}} lastUpdated:{seconds:1692063633 nanos:126508511} id:"W8UA3Qt2is9jQ5jeTguLWP" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onSystemConditionCreate": "W8UA3Qt2is9jQ5jeTguLWP"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting Creating agent disconnect with timeout 1s {"component": "alarms", "onSystemConditionCreate": "qqYBdqk7WPkARoRmrZQ6vX"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting Creating agent disconnect with timeout 1s {"component": "alarms", "onSystemConditionCreate": "W8UA3Qt2is9jQ5jeTguLWP"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting name: uxELjJ6dzuXu3EbcnWZXgM-opni-alerting + rules: + - alert: uxELjJ6dzuXu3EbcnWZXgM + expr: opni:custom:prometheus_query{opni_uuid="uxELjJ6dzuXu3EbcnWZXgM"} + for: 10s + labels: + opni_promql: uxELjJ6dzuXu3EbcnWZXgM + opni_severity: Info + opni_uuid: uxELjJ6dzuXu3EbcnWZXgM + annotations: + OpniAlarmName: agent1 sanity metrics + OpniHeader: prometheus is receiving metrics + OpniSummary: see title + opni_clusterId: agent1 + opni_fingerprint: '{{ "ALERTS_FOR_STATE{opni_uuid=\"uxELjJ6dzuXu3EbcnWZXgM\"} OR vector(0)" | query | first | value | printf "%.0f" }}' + opni_goldenSignal: Custom + opni_uuid: uxELjJ6dzuXu3EbcnWZXgM + - record: opni:custom:prometheus_query + expr: sum(up > 0) > 0 + labels: + opni_uuid: uxELjJ6dzuXu3EbcnWZXgM + {"component": "alarms", "Expr": "user-query"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting received condition update: name:"agent-capability-unhealthy" description:"Alert when some downstream agent capability becomes unhealthy" labels:"agent-capability-health" labels:"opni" labels:"_default" severity:Critical alertType:{downstreamCapability:{clusterId:{id:"agent2"} capabilityState:"Failure" capabilityState:"Pending" for:{seconds:1}}} lastUpdated:{seconds:1692063633 nanos:323886561} id:"tsMswNN8UA3NDDHLuq4mdn" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onCapabilityStatusCreate": "tsMswNN8UA3NDDHLuq4mdn"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting Creating agent capability unhealthy with timeout 1s {"component": "alarms", "onCapabilityStatusCreate": "tsMswNN8UA3NDDHLuq4mdn"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting received condition update: name:"agent-capability-unhealthy" description:"Alert when some downstream agent capability becomes unhealthy" labels:"agent-capability-health" labels:"opni" labels:"_default" severity:Critical alertType:{downstreamCapability:{clusterId:{id:"agent3"} capabilityState:"Failure" capabilityState:"Pending" for:{seconds:1}}} lastUpdated:{seconds:1692063633 nanos:527070681} id:"kgWt69xi3LtA7U5u5WgCy7" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onCapabilityStatusCreate": "kgWt69xi3LtA7U5u5WgCy7"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting name: 6WkZhz8wcBYMCCrAh2mR5L-opni-alerting + rules: + - alert: 6WkZhz8wcBYMCCrAh2mR5L + expr: opni:custom:prometheus_query{opni_uuid="6WkZhz8wcBYMCCrAh2mR5L"} + for: 10s + labels: + opni_promql: 6WkZhz8wcBYMCCrAh2mR5L + opni_severity: Info + opni_uuid: 6WkZhz8wcBYMCCrAh2mR5L + annotations: + OpniAlarmName: agent2 sanity metrics + OpniHeader: prometheus is receiving metrics + OpniSummary: see title + opni_clusterId: agent2 + opni_fingerprint: '{{ "ALERTS_FOR_STATE{opni_uuid=\"6WkZhz8wcBYMCCrAh2mR5L\"} OR vector(0)" | query | first | value | printf "%.0f" }}' + opni_goldenSignal: Custom + opni_uuid: 6WkZhz8wcBYMCCrAh2mR5L + - record: opni:custom:prometheus_query + expr: sum(up > 0) > 0 + labels: + opni_uuid: 6WkZhz8wcBYMCCrAh2mR5L + {"component": "alarms", "Expr": "user-query"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting name: s8sq5YynSQuqNEMoqwL6mQ-opni-alerting + rules: + - alert: s8sq5YynSQuqNEMoqwL6mQ + expr: opni:custom:prometheus_query{opni_uuid="s8sq5YynSQuqNEMoqwL6mQ"} + for: 10s + labels: + opni_promql: s8sq5YynSQuqNEMoqwL6mQ + opni_severity: Info + opni_uuid: s8sq5YynSQuqNEMoqwL6mQ + annotations: + OpniAlarmName: agent3 sanity metrics + OpniHeader: prometheus is receiving metrics + OpniSummary: see title + opni_clusterId: agent3 + opni_fingerprint: '{{ "ALERTS_FOR_STATE{opni_uuid=\"s8sq5YynSQuqNEMoqwL6mQ\"} OR vector(0)" | query | first | value | printf "%.0f" }}' + opni_goldenSignal: Custom + opni_uuid: s8sq5YynSQuqNEMoqwL6mQ + - record: opni:custom:prometheus_query + expr: sum(up > 0) > 0 + labels: + opni_uuid: s8sq5YynSQuqNEMoqwL6mQ + {"component": "alarms", "Expr": "user-query"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting received condition update: name:"agent-disconnect" description:"Alert when the downstream agent disconnects from the opni upstream" labels:"agent-disconnect" labels:"opni" labels:"_default" severity:Critical alertType:{system:{clusterId:{id:"agent2"} timeout:{seconds:1}}} lastUpdated:{seconds:1692063633 nanos:319151968} id:"VHHsduREaDEvBj9kcPjSNJ" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onSystemConditionCreate": "VHHsduREaDEvBj9kcPjSNJ"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting received condition update: name:"agent-capability-unhealthy" description:"Alert when some downstream agent capability becomes unhealthy" labels:"agent-capability-health" labels:"opni" labels:"_default" severity:Critical alertType:{downstreamCapability:{clusterId:{id:"agent1"} capabilityState:"Failure" capabilityState:"Pending" for:{seconds:1}}} lastUpdated:{seconds:1692063633 nanos:139779253} id:"fETVFJdkn7WrPwHMiugGSN" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onCapabilityStatusCreate": "fETVFJdkn7WrPwHMiugGSN"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting Creating agent disconnect with timeout 1s {"component": "alarms", "onSystemConditionCreate": "VHHsduREaDEvBj9kcPjSNJ"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting Creating agent capability unhealthy with timeout 1s {"component": "alarms", "onCapabilityStatusCreate": "fETVFJdkn7WrPwHMiugGSN"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting Creating agent capability unhealthy with timeout 1s {"component": "alarms", "onCapabilityStatusCreate": "kgWt69xi3LtA7U5u5WgCy7"} + 2023-08-15T01:40:33Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:33Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:33Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml"} + ts=2023-08-15T01:40:33.955Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml + ts=2023-08-15T01:40:33.955Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml"} + ts=2023-08-15T01:40:34.166Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000294759s + ts=2023-08-15T01:40:34.279461979Z caller=loader.go:117 level=warn msg="bucket index not found" user=agent2 + ts=2023-08-15T01:40:34.4027482Z caller=loader.go:117 level=warn msg="bucket index not found" user=agent1 + ts=2023-08-15T01:40:34.405248128Z caller=loader.go:117 level=warn msg="bucket index not found" user=agent3 + 2023-08-15T01:40:34Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:34Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:34Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:34Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:34Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:34Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + ts=2023-08-15T01:40:34.959Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=4.00108281s + 2023-08-15T01:40:34Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063633 nanos:216170439 {"component": "alarms", "onSystemConditionCreate": "W8UA3Qt2is9jQ5jeTguLWP"} + 2023-08-15T01:40:34Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063633 nanos:569546527 {"component": "alarms", "onSystemConditionCreate": "qqYBdqk7WPkARoRmrZQ6vX"} + 2023-08-15T01:40:34Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063633 nanos:371422007 {"component": "alarms", "onSystemConditionCreate": "VHHsduREaDEvBj9kcPjSNJ"} + 2023-08-15T01:40:34Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063633 nanos:216170439 {"component": "alarms", "onSystemConditionCreate": "W8UA3Qt2is9jQ5jeTguLWP"} + 2023-08-15T01:40:34Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063633 nanos:371422007 {"component": "alarms", "onSystemConditionCreate": "VHHsduREaDEvBj9kcPjSNJ"} + 2023-08-15T01:40:34Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063633 nanos:569546527 {"component": "alarms", "onSystemConditionCreate": "qqYBdqk7WPkARoRmrZQ6vX"} + 2023-08-15T01:40:35Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063633 nanos:371422007 {"component": "alarms", "onSystemConditionCreate": "VHHsduREaDEvBj9kcPjSNJ"} + 2023-08-15T01:40:35Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063633 nanos:216170439 {"component": "alarms", "onSystemConditionCreate": "W8UA3Qt2is9jQ5jeTguLWP"} + 2023-08-15T01:40:35Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063633 nanos:569546527 {"component": "alarms", "onSystemConditionCreate": "qqYBdqk7WPkARoRmrZQ6vX"} + 2023-08-15T01:40:35Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:35Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:35Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47382: EOF + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47248: read tcp 127.0.0.1:41355->127.0.0.1:47248: read: connection reset by peer + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47418: EOF + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47404: read tcp 127.0.0.1:41355->127.0.0.1:47404: read: connection reset by peer + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47332: read tcp 127.0.0.1:41355->127.0.0.1:47332: read: connection reset by peer + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47342: read tcp 127.0.0.1:41355->127.0.0.1:47342: read: connection reset by peer + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47352: read tcp 127.0.0.1:41355->127.0.0.1:47352: read: connection reset by peer + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47394: read tcp 127.0.0.1:41355->127.0.0.1:47394: read: connection reset by peer + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47420: read tcp 127.0.0.1:41355->127.0.0.1:47420: read: connection reset by peer + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47316: read tcp 127.0.0.1:41355->127.0.0.1:47316: read: connection reset by peer + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47368: read tcp 127.0.0.1:41355->127.0.0.1:47368: read: connection reset by peer + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47460: read tcp 127.0.0.1:41355->127.0.0.1:47460: read: connection reset by peer + 2023-08-15T01:40:35Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:35Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:35Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:35Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:35Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:35Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:35Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:35Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:36Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:36Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:36Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:36 http: TLS handshake error from 127.0.0.1:47536: read tcp 127.0.0.1:41355->127.0.0.1:47536: read: connection reset by peer + 2023/08/15 01:40:36 http: TLS handshake error from 127.0.0.1:47670: EOF + 2023/08/15 01:40:36 http: TLS handshake error from 127.0.0.1:47646: EOF + 2023-08-15T01:40:36Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:36Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:36Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:36Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:36Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:36Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:36Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:36Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:36Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:36Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:36Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + ts=2023-08-15T01:40:36.960Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=6.001822481s + 2023/08/15 01:40:36 http: TLS handshake error from 127.0.0.1:47738: EOF + 2023/08/15 01:40:36 http: TLS handshake error from 127.0.0.1:47764: read tcp 127.0.0.1:41355->127.0.0.1:47764: read: connection reset by peer + 2023/08/15 01:40:36 http: TLS handshake error from 127.0.0.1:47776: read tcp 127.0.0.1:41355->127.0.0.1:47776: read: connection reset by peer + 2023-08-15T01:40:37Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:37Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:37Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:37Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:37Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:37Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:37Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:37Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:37Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:37Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:37Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:37 http: TLS handshake error from 127.0.0.1:47862: read tcp 127.0.0.1:41355->127.0.0.1:47862: read: connection reset by peer + 2023/08/15 01:40:37 http: TLS handshake error from 127.0.0.1:47896: EOF + 2023/08/15 01:40:37 http: TLS handshake error from 127.0.0.1:47914: read tcp 127.0.0.1:41355->127.0.0.1:47914: read: connection reset by peer + 2023/08/15 01:40:37 http: TLS handshake error from 127.0.0.1:48016: read tcp 127.0.0.1:41355->127.0.0.1:48016: read: connection reset by peer + 2023-08-15T01:40:38Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:38Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:38Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:38Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:38Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:38Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:38Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:38Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + ts=2023-08-15T01:40:38.961Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=8.002725938s + 2023/08/15 01:40:38 http: TLS handshake error from 127.0.0.1:48038: read tcp 127.0.0.1:41355->127.0.0.1:48038: read: connection reset by peer + 2023-08-15T01:40:38Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:38Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:38Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:38 http: TLS handshake error from 127.0.0.1:48100: EOF + 2023/08/15 01:40:38 http: TLS handshake error from 127.0.0.1:48114: EOF + 2023/08/15 01:40:39 http: TLS handshake error from 127.0.0.1:48228: EOF + 2023/08/15 01:40:39 http: TLS handshake error from 127.0.0.1:48218: read tcp 127.0.0.1:41355->127.0.0.1:48218: read: connection reset by peer + 2023-08-15T01:40:39Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:39Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:39Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:39Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:39Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:39Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:39Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:39Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:39Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:39Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:39Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:39 http: TLS handshake error from 127.0.0.1:48230: EOF + 2023/08/15 01:40:39 http: TLS handshake error from 127.0.0.1:48306: EOF + 2023/08/15 01:40:39 http: TLS handshake error from 127.0.0.1:48362: EOF + 2023/08/15 01:40:39 http: TLS handshake error from 127.0.0.1:48358: read tcp 127.0.0.1:41355->127.0.0.1:48358: read: connection reset by peer + 2023/08/15 01:40:39 http: TLS handshake error from 127.0.0.1:48374: EOF + 2023/08/15 01:40:39 http: TLS handshake error from 127.0.0.1:48346: EOF + 2023-08-15T01:40:40Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:40Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:40Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:40Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:40Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:40Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:40Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:40Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:40Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:40Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:40Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + ts=2023-08-15T01:40:40.962Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003741909s + 2023-08-15T01:40:40Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:40Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:41Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:41Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:41Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:41Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:41Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:41Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:41Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:41Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:41Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:41 http: TLS handshake error from 127.0.0.1:58026: EOF + 2023/08/15 01:40:41 http: TLS handshake error from 127.0.0.1:58090: read tcp 127.0.0.1:41355->127.0.0.1:58090: read: connection reset by peer + 2023/08/15 01:40:41 http: TLS handshake error from 127.0.0.1:58106: read tcp 127.0.0.1:41355->127.0.0.1:58106: read: connection reset by peer + 2023/08/15 01:40:41 http: TLS handshake error from 127.0.0.1:58154: read tcp 127.0.0.1:41355->127.0.0.1:58154: read: connection reset by peer + 2023/08/15 01:40:41 http: TLS handshake error from 127.0.0.1:58226: EOF + 2023/08/15 01:40:41 http: TLS handshake error from 127.0.0.1:58214: read tcp 127.0.0.1:41355->127.0.0.1:58214: read: connection reset by peer + 2023/08/15 01:40:41 http: TLS handshake error from 127.0.0.1:58110: read tcp 127.0.0.1:41355->127.0.0.1:58110: read: connection reset by peer + 2023/08/15 01:40:41 http: TLS handshake error from 127.0.0.1:58138: read tcp 127.0.0.1:41355->127.0.0.1:58138: read: connection reset by peer + 2023/08/15 01:40:42 http: TLS handshake error from 127.0.0.1:58134: EOF + 2023/08/15 01:40:42 http: TLS handshake error from 127.0.0.1:58188: read tcp 127.0.0.1:41355->127.0.0.1:58188: read: connection reset by peer + 2023/08/15 01:40:42 http: TLS handshake error from 127.0.0.1:58126: EOF + 2023/08/15 01:40:42 http: TLS handshake error from 127.0.0.1:58184: EOF + 2023/08/15 01:40:42 http: TLS handshake error from 127.0.0.1:58132: read tcp 127.0.0.1:41355->127.0.0.1:58132: read: connection reset by peer + 2023-08-15T01:40:42Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:42Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + ts=2023-08-15T01:40:42.168Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001481355s + 2023-08-15T01:40:42Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:42Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:42Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:42Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:42Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:42Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:42Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:42Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:42Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:42 http: TLS handshake error from 127.0.0.1:58228: read tcp 127.0.0.1:41355->127.0.0.1:58228: read: connection reset by peer + 2023/08/15 01:40:42 http: TLS handshake error from 127.0.0.1:58240: read tcp 127.0.0.1:41355->127.0.0.1:58240: read: connection reset by peer + 2023/08/15 01:40:42 http: TLS handshake error from 127.0.0.1:58266: EOF + 2023/08/15 01:40:43 http: TLS handshake error from 127.0.0.1:58318: EOF + 2023/08/15 01:40:43 http: TLS handshake error from 127.0.0.1:58354: EOF + 2023-08-15T01:40:43Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:43Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:43Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:43Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:43Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:43Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:43Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:41759 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46699", "scheme": "http"} + 2023-08-15T01:40:43Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:40117 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:40937", "scheme": "http"} + 2023-08-15T01:40:43Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:41759 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46699", "scheme": "http"} + 2023-08-15T01:40:43Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:40117 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:40937", "scheme": "http"} + 2023-08-15T01:40:43Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:42355 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46631", "scheme": "http"} + 2023-08-15T01:40:43Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:42355 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46631", "scheme": "http"} + 2023-08-15T01:40:43Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:43Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:43Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:43Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:43Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:43 http: TLS handshake error from 127.0.0.1:58412: EOF + 2023/08/15 01:40:43 http: TLS handshake error from 127.0.0.1:58406: EOF + 2023/08/15 01:40:43 http: TLS handshake error from 127.0.0.1:58426: EOF + 2023/08/15 01:40:43 http: TLS handshake error from 127.0.0.1:58458: read tcp 127.0.0.1:41355->127.0.0.1:58458: read: connection reset by peer + 2023/08/15 01:40:43 http: TLS handshake error from 127.0.0.1:58414: read tcp 127.0.0.1:41355->127.0.0.1:58414: read: connection reset by peer + 2023-08-15T01:40:44Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:44Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:44Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:44Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:44Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:44Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:44Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:44Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023/08/15 01:40:44 http: TLS handshake error from 127.0.0.1:58524: read tcp 127.0.0.1:41355->127.0.0.1:58524: read: connection reset by peer + 2023-08-15T01:40:44Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:44Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:44Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:44 http: TLS handshake error from 127.0.0.1:58560: EOF + 2023/08/15 01:40:44 http: TLS handshake error from 127.0.0.1:58544: read tcp 127.0.0.1:41355->127.0.0.1:58544: read: connection reset by peer + 2023/08/15 01:40:44 http: TLS handshake error from 127.0.0.1:58574: read tcp 127.0.0.1:41355->127.0.0.1:58574: read: connection reset by peer + 2023/08/15 01:40:44 http: TLS handshake error from 127.0.0.1:58542: EOF + 2023/08/15 01:40:44 http: TLS handshake error from 127.0.0.1:58670: read tcp 127.0.0.1:41355->127.0.0.1:58670: read: connection reset by peer + 2023/08/15 01:40:44 http: TLS handshake error from 127.0.0.1:58602: read tcp 127.0.0.1:41355->127.0.0.1:58602: read: connection reset by peer + 2023/08/15 01:40:44 http: TLS handshake error from 127.0.0.1:58626: EOF + 2023-08-15T01:40:44Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:44Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:45Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:45Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:45Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:45Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:45Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:45Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023/08/15 01:40:45 http: TLS handshake error from 127.0.0.1:58694: EOF + 2023-08-15T01:40:45Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:45Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:45Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:40:46Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:46Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:46Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:46Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:46Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:46Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:46Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:46Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:46Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:46Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:46Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:46 http: TLS handshake error from 127.0.0.1:58950: EOF + 2023/08/15 01:40:46 http: TLS handshake error from 127.0.0.1:58936: read tcp 127.0.0.1:41355->127.0.0.1:58936: read: connection reset by peer + 2023/08/15 01:40:46 http: TLS handshake error from 127.0.0.1:59022: EOF + 2023/08/15 01:40:46 http: TLS handshake error from 127.0.0.1:59034: EOF + 2023/08/15 01:40:47 http: TLS handshake error from 127.0.0.1:59016: read tcp 127.0.0.1:41355->127.0.0.1:59016: read: connection reset by peer + 2023-08-15T01:40:47Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:47Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:47Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:47Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:47Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:47Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:47Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:47Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:47Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:47Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:47Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:47 http: TLS handshake error from 127.0.0.1:59076: EOF + 2023/08/15 01:40:47 http: TLS handshake error from 127.0.0.1:59090: EOF + 2023/08/15 01:40:47 http: TLS handshake error from 127.0.0.1:59140: read tcp 127.0.0.1:41355->127.0.0.1:59140: read: connection reset by peer + 2023/08/15 01:40:47 http: TLS handshake error from 127.0.0.1:59128: read tcp 127.0.0.1:41355->127.0.0.1:59128: read: connection reset by peer + 2023/08/15 01:40:48 http: TLS handshake error from 127.0.0.1:59260: EOF + 2023-08-15T01:40:48Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:48Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:48Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:48Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:48Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:48Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:48Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:48Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:40117 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:40937", "scheme": "http"} + 2023-08-15T01:40:48Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:48Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:41759 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46699", "scheme": "http"} + 2023-08-15T01:40:48Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:42355 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46631", "scheme": "http"} + 2023-08-15T01:40:48Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:48Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:48Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:49 http: TLS handshake error from 127.0.0.1:59372: read tcp 127.0.0.1:41355->127.0.0.1:59372: read: connection reset by peer + 2023/08/15 01:40:49 http: TLS handshake error from 127.0.0.1:59416: EOF + 2023-08-15T01:40:49Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:49Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:49Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:49Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:49Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:49Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:49Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:49Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:49Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:49Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:49Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:49 http: TLS handshake error from 127.0.0.1:59480: EOF + 2023/08/15 01:40:49 http: TLS handshake error from 127.0.0.1:59486: EOF + 2023/08/15 01:40:49 http: TLS handshake error from 127.0.0.1:59496: EOF + 2023/08/15 01:40:49 http: TLS handshake error from 127.0.0.1:59490: EOF + 2023/08/15 01:40:49 http: TLS handshake error from 127.0.0.1:59562: read tcp 127.0.0.1:41355->127.0.0.1:59562: read: connection reset by peer + 2023/08/15 01:40:49 http: TLS handshake error from 127.0.0.1:59548: EOF + 2023/08/15 01:40:50 http: TLS handshake error from 127.0.0.1:59632: read tcp 127.0.0.1:41355->127.0.0.1:59632: read: connection reset by peer + 2023/08/15 01:40:50 http: TLS handshake error from 127.0.0.1:59628: read tcp 127.0.0.1:41355->127.0.0.1:59628: read: connection reset by peer + 2023-08-15T01:40:50Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:50Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:50Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:50Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:50Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:50Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:50Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:50Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:50Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:50Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:50Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:50 http: TLS handshake error from 127.0.0.1:59756: read tcp 127.0.0.1:41355->127.0.0.1:59756: read: connection reset by peer + 2023/08/15 01:40:50 http: TLS handshake error from 127.0.0.1:59776: EOF + 2023/08/15 01:40:50 http: TLS handshake error from 127.0.0.1:59732: read tcp 127.0.0.1:41355->127.0.0.1:59732: read: connection reset by peer + 2023/08/15 01:40:50 http: TLS handshake error from 127.0.0.1:59716: EOF + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:59810: read tcp 127.0.0.1:41355->127.0.0.1:59810: read: connection reset by peer + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:59874: EOF + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:59842: EOF + 2023-08-15T01:40:51Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:51Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:51Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:51Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:51Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:51Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:51Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:51Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:51Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:50392: EOF + 2023-08-15T01:40:51Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:51Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:50380: EOF + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:50374: EOF + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:50408: EOF + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:50540: EOF + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:50504: EOF + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:50508: EOF + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:50422: read tcp 127.0.0.1:41355->127.0.0.1:50422: read: connection reset by peer + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:50452: read tcp 127.0.0.1:41355->127.0.0.1:50452: read: connection reset by peer + 2023/08/15 01:40:52 http: TLS handshake error from 127.0.0.1:50556: EOF + 2023-08-15T01:40:52Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:52Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:52Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:52Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:52Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:52Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:52Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:52Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:52Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:52Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:52Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:52 http: TLS handshake error from 127.0.0.1:50582: read tcp 127.0.0.1:41355->127.0.0.1:50582: read: connection reset by peer + 2023/08/15 01:40:52 http: TLS handshake error from 127.0.0.1:50604: read tcp 127.0.0.1:41355->127.0.0.1:50604: read: connection reset by peer + 2023/08/15 01:40:52 http: TLS handshake error from 127.0.0.1:50618: EOF + 2023/08/15 01:40:52 http: TLS handshake error from 127.0.0.1:50622: read tcp 127.0.0.1:41355->127.0.0.1:50622: read: connection reset by peer + 2023/08/15 01:40:52 http: TLS handshake error from 127.0.0.1:50650: EOF + 2023-08-15T01:40:53Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:53Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:53Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:53Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:53Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:53Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:53Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:41759 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46699", "scheme": "http"} + 2023-08-15T01:40:53Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:40117 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:40937", "scheme": "http"} + 2023-08-15T01:40:53Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:42355 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46631", "scheme": "http"} + 2023-08-15T01:40:53Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:53Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:53Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023/08/15 01:40:53 http: TLS handshake error from 127.0.0.1:50770: EOF + 2023-08-15T01:40:53Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:53Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:53 http: TLS handshake error from 127.0.0.1:50828: EOF + 2023/08/15 01:40:53 http: TLS handshake error from 127.0.0.1:50840: read tcp 127.0.0.1:41355->127.0.0.1:50840: read: connection reset by peer + 2023-08-15T01:40:54Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:54Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:54Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:40117 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:40937", "scheme": "http"} + 2023-08-15T01:40:54Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:41759 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46699", "scheme": "http"} + 2023-08-15T01:40:54Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:42355 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46631", "scheme": "http"} + 2023-08-15T01:40:54Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:54Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:54Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:54Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:54Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:54Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:54Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:50960: EOF + 2023-08-15T01:40:54Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:54Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:50998: read tcp 127.0.0.1:41355->127.0.0.1:50998: read: connection reset by peer + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:50978: EOF + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:51026: read tcp 127.0.0.1:41355->127.0.0.1:51026: read: connection reset by peer + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:51020: EOF + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:51078: EOF + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:51076: read tcp 127.0.0.1:41355->127.0.0.1:51076: read: connection reset by peer + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:51100: EOF + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:51050: EOF + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:51104: EOF + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:51134: EOF + 2023-08-15T01:40:55Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:55Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:55Z DEBUG api => {"method": "POST", "path": "/plugin_alerting/alertmanager/api/v2/*any", "to": "127.0.0.1:41281 (plugin_alerting)", "for": "127.0.0.1", "host": "127.0.0.1:45615", "scheme": "https"} + 2023-08-15T01:40:55Z DEBUG api => {"method": "POST", "path": "/plugin_alerting/alertmanager/api/v2/*any", "to": "127.0.0.1:41281 (plugin_alerting)", "for": "127.0.0.1", "host": "127.0.0.1:45615", "scheme": "https"} + 2023-08-15T01:40:55Z DEBUG api => {"method": "POST", "path": "/plugin_alerting/alertmanager/api/v2/*any", "to": "127.0.0.1:41281 (plugin_alerting)", "for": "127.0.0.1", "host": "127.0.0.1:45615", "scheme": "https"} +2023/08/15 01:40:55 http: TLS handshake error from 127.0.0.1:45752: EOF + 2023-08-15T01:40:55Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:55Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:55Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:55Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:55Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:55Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:55Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:55Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:55Z ERROR plugin.metrics.cortex-admin failed to get cluster status {"error": "rpc error: code = Internal desc = Get \"https://localhost:41477/services\": dial tcp 127.0.0.1:41477: connect: connection refused"} + 2023-08-15T01:40:55Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:55Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:55Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:55Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:55Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:55 http: TLS handshake error from 127.0.0.1:51236: EOF + 2023/08/15 01:40:55 http: TLS handshake error from 127.0.0.1:51216: EOF + 2023/08/15 01:40:55 http: TLS handshake error from 127.0.0.1:51280: read tcp 127.0.0.1:41355->127.0.0.1:51280: read: connection reset by peer + 2023/08/15 01:40:55 http: TLS handshake error from 127.0.0.1:51306: read tcp 127.0.0.1:41355->127.0.0.1:51306: read: connection reset by peer + 2023/08/15 01:40:56 http: TLS handshake error from 127.0.0.1:51358: EOF + 2023/08/15 01:40:56 http: TLS handshake error from 127.0.0.1:51342: EOF + 2023/08/15 01:40:56 http: TLS handshake error from 127.0.0.1:51308: EOF + 2023-08-15T01:40:56Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:56Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + ts=2023-08-15T01:40:56.300Z caller=notify.go:740 level=warn component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 127.0.0.1:3000: connect: connection refused" + ts=2023-08-15T01:40:56.413Z caller=notify.go:740 level=warn component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 127.0.0.1:3000: connect: connection refused" + ts=2023-08-15T01:40:56.415Z caller=notify.go:740 level=warn component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 127.0.0.1:3000: connect: connection refused" + 2023-08-15T01:40:56Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:56Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:56Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:56Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:56Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "W8UA3Qt2is9jQ5jeTguLWP"} + 2023-08-15T01:40:56Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onCapabilityStatusCreate": "tsMswNN8UA3NDDHLuq4mdn"} + 2023-08-15T01:40:56Z INFO plugin.alerting shutting down cluster driver update handler + 2023-08-15T01:40:56Z INFO plugin.alerting exiting main sync loop + 2023-08-15T01:40:56Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "6e33b99a-7400-45fd-8d69-2d0f7dddaf82"} + 2023-08-15T01:40:56Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onCapabilityStatusCreate": "kgWt69xi3LtA7U5u5WgCy7"} + 2023-08-15T01:40:56Z DEBUG plugin.alerting closing cortex cluster status watcher... {"watcher": "cortex-cluster-status"} + 2023-08-15T01:40:56Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "qqYBdqk7WPkARoRmrZQ6vX"} + 2023-08-15T01:40:56Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "bd425fda-bda6-4fee-ad58-e7b42139b468"} + 2023-08-15T01:40:56Z INFO gateway shutting down plugins + 2023-08-15T01:40:56Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "W8UA3Qt2is9jQ5jeTguLWP"} + 2023-08-15T01:40:56Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "VHHsduREaDEvBj9kcPjSNJ"} + 2023-08-15T01:40:56Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "qqYBdqk7WPkARoRmrZQ6vX"} + 2023-08-15T01:40:56Z WARN jetstream disconnected from jetstream + 2023-08-15T01:40:56Z WARN plugin.metrics rule discovery stream closing {"error": "context canceled"} + 2023-08-15T01:40:56Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onCapabilityStatusCreate": "fETVFJdkn7WrPwHMiugGSN"} + 2023-08-15T01:40:56Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "VHHsduREaDEvBj9kcPjSNJ"} + 2023-08-15T01:40:56Z DEBUG plugin.metrics rule discovery stream closing {"error": "context canceled"} + 2023-08-15T01:40:56Z WARN plugin.alerting failed to receive cluster health status from grpc stream, retrying... + 2023-08-15T01:40:56Z WARN plugin.metrics rule discovery stream closing {"error": "context canceled"} + 2023-08-15T01:40:56Z DEBUG plugin.metrics rule discovery stream closing {"error": "context canceled"} + 2023-08-15T01:40:56Z INFO test.env Cortex stopping... + 2023-08-15T01:40:56Z WARN plugin.metrics rule discovery stream closing {"error": "context canceled"} + 2023-08-15T01:40:56Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z DEBUG plugin.metrics rule discovery stream closing {"error": "context canceled"} + 2023-08-15T01:40:56Z ERROR plugin.alerting failed to receive cluster event : rpc error: code = Canceled desc = context canceled + 2023-08-15T01:40:56Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:40:56Z ERROR test agent "agent2" exited: context canceled + 2023-08-15T01:40:56Z WARN jetstream disconnected from jetstream + 2023-08-15T01:40:56Z WARN jetstream disconnected from jetstream + 2023-08-15T01:40:56Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:40:56Z INFO gateway all plugins shut down + 2023-08-15T01:40:56Z ERROR test agent "agent3" exited: context canceled + 2023-08-15T01:40:56Z WARN test.env management server exited with error {"error": "context canceled"} + 2023-08-15T01:40:56Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:40:56Z DEBUG gateway.delegate agent disconnected {"id": "agent1"} + 2023-08-15T01:40:56Z DEBUG gateway.sync agent disconnected {"id": "agent1"} + 2023-08-15T01:40:56Z INFO gateway.grpc agent stream closing {"error": "context canceled"} + 2023-08-15T01:40:56Z DEBUG gateway.delegate agent disconnected {"id": "agent3"} + 2023-08-15T01:40:56Z INFO gateway.grpc agent stream closing {"error": "context canceled"} + 2023-08-15T01:40:56Z ERROR test agent "agent1" exited: context canceled + 2023-08-15T01:40:56Z DEBUG gateway.sync agent disconnected {"id": "agent2"} + 2023-08-15T01:40:56Z WARN gateway http server exited with error {"error": "context canceled"} + 2023-08-15T01:40:56Z INFO gateway.grpc agent stream closing {"error": "context canceled"} + 2023-08-15T01:40:56Z DEBUG gateway.sync agent disconnected {"id": "agent3"} + 2023-08-15T01:40:56Z DEBUG gateway.delegate agent disconnected {"id": "agent2"} + 2023-08-15T01:40:56Z WARN test.env gateway server exited with error {"error": "context canceled"} + ts=2023-08-15T01:40:56.793Z caller=alertmanager_main.go:601 level=info msg="Received SIGTERM, exiting gracefully..." + 2023-08-15T01:40:56Z ERROR plugin.alerting read tcp 127.0.0.1:49688->127.0.0.1:32799: read: connection reset by peer + 2023-08-15T01:40:56Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error + 2023-08-15T01:40:56Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error + 2023-08-15T01:40:56Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} +• [23.076 seconds] +------------------------------ +Invalidated and clean up suite test when we update an alarm of the disconnect type should successfully update the group of the alarm [integration] +/home/ec2-user/opni/test/plugins/alerting/delete_test.go:154 + 2023-08-15T01:40:56Z INFO test.env Starting test environment + 2023-08-15T01:40:56Z INFO test.env Copying 7 files from embedded testdata/cortex to /tmp/opni-test-522595029/cortex + 2023-08-15T01:40:56Z INFO test.env Waiting for jetstream to start... + 2023-08-15T01:40:56Z INFO test.env Cortex stopped + [231646] 2023/08/15 01:40:56.819620 [INF] Starting nats-server + [231646] 2023/08/15 01:40:56.819685 [INF] Version: 2.9.21 + [231646] 2023/08/15 01:40:56.819689 [INF] Git: [b2e7725] + [231646] 2023/08/15 01:40:56.819696 [INF] Name: NBEONMMZHXYZAX6W2XLOBAQX7R4DWEQ27VEUM257CW53JY5AKV6D3QNU + [231646] 2023/08/15 01:40:56.819701 [INF] Node: 86Wk6YxY + [231646] 2023/08/15 01:40:56.819704 [INF] ID: NBEONMMZHXYZAX6W2XLOBAQX7R4DWEQ27VEUM257CW53JY5AKV6D3QNU + [231646] 2023/08/15 01:40:56.819720 [INF] Using configuration file: /tmp/opni-test-522595029/jetstream/jetstream.conf + [231646] 2023/08/15 01:40:56.820496 [INF] Starting JetStream + [231646] 2023/08/15 01:40:56.820638 [INF] _ ___ _____ ___ _____ ___ ___ _ __ __ + [231646] 2023/08/15 01:40:56.820654 [INF] _ | | __|_ _/ __|_ _| _ \ __| /_\ | \/ | + [231646] 2023/08/15 01:40:56.820658 [INF] | || | _| | | \__ \ | | | / _| / _ \| |\/| | + [231646] 2023/08/15 01:40:56.820661 [INF] \__/|___| |_| |___/ |_| |_|_\___/_/ \_\_| |_| + [231646] 2023/08/15 01:40:56.820664 [INF] + [231646] 2023/08/15 01:40:56.820667 [INF] https://docs.nats.io/jetstream + [231646] 2023/08/15 01:40:56.820670 [INF] + [231646] 2023/08/15 01:40:56.820673 [INF] ---------------- JETSTREAM ---------------- + [231646] 2023/08/15 01:40:56.820679 [INF] Max Memory: 11.71 GB + [231646] 2023/08/15 01:40:56.820683 [INF] Max Storage: 5.29 GB + [231646] 2023/08/15 01:40:56.820686 [INF] Store Directory: "/tmp/opni-test-522595029/jetstream/data/jetstream" + [231646] 2023/08/15 01:40:56.820689 [INF] ------------------------------------------- + [231646] 2023/08/15 01:40:56.821056 [INF] Listening for client connections on 0.0.0.0:42403 + [231646] 2023/08/15 01:40:56.821388 [INF] Server is ready + 2023-08-15T01:40:56Z INFO test.env Jetstream started + 2023-08-15T01:40:56Z DEBUG keyring loaded ephemeral key {"path": "/tmp/opni-test-522595029/keyring/local-agent.json", "usage": "auth", "labels": {"opni.io/session-attribute":"local"}} + 2023-08-15T01:40:56Z DEBUG gateway found 0 plugins + 2023-08-15T01:40:56Z DEBUG gateway loaded plugin manifest {"plugins": 0} + 2023-08-15T01:40:56Z INFO gateway.cache compressing and archiving plugins... + 2023-08-15T01:40:56Z DEBUG gateway.cache added 0 new plugins to cache + 2023-08-15T01:40:56Z INFO gateway running plugin cache gc + 2023-08-15T01:40:56Z INFO gateway.update-server registering update handler for strategy "binary" + 2023-08-15T01:40:56Z INFO gateway.update-server registering update handler for strategy "kubernetes" + 2023-08-15T01:40:56Z INFO gateway.update-server registering update handler for strategy "noop" + 2023-08-15T01:40:56Z DEBUG gateway.grpc registering service {"service": "control.HealthListener"} + 2023-08-15T01:40:56Z DEBUG gateway.grpc registering internal service {"service": "stream.Delegate"} + 2023-08-15T01:40:56Z DEBUG gateway.grpc registering service {"service": "core.Pinger"} + 2023-08-15T01:40:56Z INFO test.env Loading gateway plugins... + 2023-08-15T01:40:56Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.apiextensions.HTTPAPIExtension", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.ManagementAPIExtension", "opni.backends.Capability", "opni.Metrics", "opni.System"]} + 2023-08-15T01:40:56Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.apiextensions.ManagementAPIExtension", "opni.Metrics", "opni.backends.Capability", "opni.apiextensions.StreamAPIExtension", "opni.System", "opni.apiextensions.HTTPAPIExtension"]} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.System"} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_alerting"} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.System"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.grpc plugin rejected internal stream connection {"plugin": "plugin_alerting"} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_metrics"} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG plugin.metrics/pkg/gateway.stream internal gateway stream connected + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/trace", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/profile", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/block", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/heap", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/ready", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/healthy", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service cortexadmin.CortexAdmin + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service alerting.AlertConditions + 2023-08-15T01:40:56Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/metrics", "capability": "metrics"} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Metrics"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Metrics"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "cortexadmin.CortexAdmin"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/AllUserStats"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/WriteMetrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/Query"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/QueryRange"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetRule"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricMetadata"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ListRules"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/LoadRules"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/DeleteRule"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/FlushBlocks"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetSeriesMetrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricLabelSets"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexStatus"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexConfig"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ExtractRawSeries"} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "cortexadmin.CortexAdmin", "rules": 17} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "cortexadmin.CortexAdmin", "rules": [{"http":{"Pattern":{"Get":"/all_user_stats"}},"method":{"name":"AllUserStats","input_type":".google.protobuf.Empty","output_type":".cortexadmin.UserIDStatsList","options":{}}},{"http":{"Pattern":{"Post":"/write_metrics"},"body":"*"},"method":{"name":"WriteMetrics","input_type":".cortexadmin.WriteRequest","output_type":".cortexadmin.WriteResponse","options":{}}},{"http":{"Pattern":{"Get":"/query"},"additional_bindings":[{"Pattern":{"Post":"/query"},"body":"*"}]},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query"},"body":"*"},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/query_range"},"additional_bindings":[{"Pattern":{"Post":"/query_range"},"body":"*"}]},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query_range"},"body":"*"},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/rules/{namespace}/{groupName}"}},"method":{"name":"GetRule","input_type":".cortexadmin.GetRuleRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/metadata"}},"method":{"name":"GetMetricMetadata","input_type":".cortexadmin.MetricMetadataRequest","output_type":".cortexadmin.MetricMetadata","options":{}}},{"http":{"Pattern":{"Get":"/rules"}},"method":{"name":"ListRules","input_type":".cortexadmin.ListRulesRequest","output_type":".cortexadmin.ListRulesResponse","options":{}}},{"http":{"Pattern":{"Post":"/rules"}},"method":{"name":"LoadRules","input_type":".cortexadmin.LoadRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/rules/{groupName}"}},"method":{"name":"DeleteRule","input_type":".cortexadmin.DeleteRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/flush_blocks"}},"method":{"name":"FlushBlocks","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/series/metadata"}},"method":{"name":"GetSeriesMetrics","input_type":".cortexadmin.SeriesRequest","output_type":".cortexadmin.SeriesInfoList","options":{}}},{"http":{"Pattern":{"Get":"/series/labels"}},"method":{"name":"GetMetricLabelSets","input_type":".cortexadmin.LabelRequest","output_type":".cortexadmin.MetricLabels","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetCortexStatus","input_type":".google.protobuf.Empty","output_type":".cortexadmin.CortexStatus","options":{}}},{"http":{"Pattern":{"Get":"/config"}},"method":{"name":"GetCortexConfig","input_type":".cortexadmin.ConfigRequest","output_type":".cortexadmin.ConfigResponse","options":{}}},{"http":{"Pattern":{"Get":"/series/raw"}},"method":{"name":"ExtractRawSeries","input_type":".cortexadmin.MatcherRequest","output_type":".cortexadmin.QueryResponse","options":{}}}]} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service cortexops.CortexOps + 2023-08-15T01:40:56Z DEBUG plugin.metrics/pkg/gateway.stream calling client handler + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/alerting", "capability": "alerting"} + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "cortexops.CortexOps"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterConfiguration"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexops.CortexOps/ConfigureCluster"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterStatus"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexops.CortexOps/UninstallCluster"} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "cortexops.CortexOps", "rules": 4} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "cortexops.CortexOps", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".cortexops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".cortexops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".cortexops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service remoteread.RemoteReadGateway + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "alerting.AlertConditions"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionGroups"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CreateAlertCondition"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/GetAlertCondition"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditions"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/UpdateAlertCondition"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionChoices"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeleteAlertCondition"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/AlertConditionStatus"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionsWithStatus"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CloneTo"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ActivateSilence"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeactivateSilence"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/Timeline"} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "alerting.AlertConditions", "rules": 13} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "alerting.AlertConditions", "rules": [{"http":{"Pattern":{"Get":"/groups"}},"method":{"name":"ListAlertConditionGroups","input_type":".google.protobuf.Empty","output_type":".core.ReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertCondition","input_type":".alerting.AlertCondition","output_type":".alerting.ConditionReference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertCondition","input_type":".alerting.ConditionReference","output_type":".alerting.AlertCondition","options":{}}},{"http":{"Pattern":{"Post":"/list"},"body":"*"},"method":{"name":"ListAlertConditions","input_type":".alerting.ListAlertConditionRequest","output_type":".alerting.AlertConditionList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertCondition","input_type":".alerting.UpdateAlertConditionRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/choices"}},"method":{"name":"ListAlertConditionChoices","input_type":".alerting.AlertDetailChoicesRequest","output_type":".alerting.ListAlertTypeDetails","options":{}}},{"http":{"Pattern":{"Delete":"/configure"}},"method":{"name":"DeleteAlertCondition","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status/{id}"}},"method":{"name":"AlertConditionStatus","input_type":".alerting.ConditionReference","output_type":".alerting.AlertStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/list/withStatus"},"body":"*"},"method":{"name":"ListAlertConditionsWithStatus","input_type":".alerting.ListStatusRequest","output_type":".alerting.ListStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/clone"},"body":"*"},"method":{"name":"CloneTo","input_type":".alerting.CloneToRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/silences"},"body":"*"},"method":{"name":"ActivateSilence","input_type":".alerting.SilenceRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/silences"}},"method":{"name":"DeactivateSilence","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/timeline"}},"method":{"name":"Timeline","input_type":".alerting.TimelineRequest","output_type":".alerting.TimelineResponse","options":{}}}]} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service alerting.AlertEndpoints + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "alerting.AlertEndpoints"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/CreateAlertEndpoint"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/GetAlertEndpoint"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ToggleNotifications"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ListAlertEndpoints"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/UpdateAlertEndpoint"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/DeleteAlertEndpoint"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/TestAlertEndpoint"} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "alerting.AlertEndpoints", "rules": 7} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "alerting.AlertEndpoints", "rules": [{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertEndpoint","input_type":".alerting.AlertEndpoint","output_type":".core.Reference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertEndpoint","input_type":".core.Reference","output_type":".alerting.AlertEndpoint","options":{}}},{"http":{"Pattern":{"Post":"/toggleNotifications"},"body":"*"},"method":{"name":"ToggleNotifications","input_type":".alerting.ToggleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/list"}},"method":{"name":"ListAlertEndpoints","input_type":".alerting.ListAlertEndpointsRequest","output_type":".alerting.AlertEndpointList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertEndpoint","input_type":".alerting.UpdateAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/delete"},"body":"*"},"method":{"name":"DeleteAlertEndpoint","input_type":".alerting.DeleteAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/test"},"body":"*"},"method":{"name":"TestAlertEndpoint","input_type":".alerting.TestAlertEndpointRequest","output_type":".alerting.TestAlertEndpointResponse","options":{}}}]} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service alerting.AlertNotifications + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "alerting.AlertNotifications"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/TriggerAlerts"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ResolveAlerts"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/PushNotification"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListNotifications"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListAlarmMessages"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListRoutingRelationships"} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "alerting.AlertNotifications", "rules": 4} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "alerting.AlertNotifications", "rules": [{"http":{"Pattern":{"Post":"/trigger"},"body":"*"},"method":{"name":"TriggerAlerts","input_type":".alerting.TriggerAlertsRequest","output_type":".alerting.TriggerAlertsResponse","options":{}}},{"http":{"Pattern":{"Post":"/push"},"body":"*"},"method":{"name":"PushNotification","input_type":".alerting.Notification","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/notifications/list"},"body":"*"},"method":{"name":"ListNotifications","input_type":".alerting.ListNotificationRequest","output_type":".alerting.ListMessageResponse","options":{}}},{"http":{"Pattern":{"Post":"/alarms/list"},"body":"*"},"method":{"name":"ListAlarmMessages","input_type":".alerting.ListAlarmMessageRequest","output_type":".alerting.ListMessageResponse","options":{}}}]} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service alerting.ops.AlertingAdmin + 2023-08-15T01:40:56Z INFO plugin.metrics.cortex-http configuring http api server + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "remoteread.RemoteReadGateway"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/AddTarget"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/EditTarget"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/RemoveTarget"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/ListTargets"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Start"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Stop"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/GetTargetStatus"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Discover"} + 2023-08-15T01:40:56Z INFO plugin.alerting using cluster driver {"driver": "test-environment"} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "remoteread.RemoteReadGateway", "rules": 8} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "remoteread.RemoteReadGateway", "rules": [{"http":{"Pattern":{"Post":"/targets"},"body":"*"},"method":{"name":"AddTarget","input_type":".remoteread.TargetAddRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Put":"/targets"},"body":"*"},"method":{"name":"EditTarget","input_type":".remoteread.TargetEditRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/targets/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"RemoveTarget","input_type":".remoteread.TargetRemoveRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/targets"}},"method":{"name":"ListTargets","input_type":".remoteread.TargetListRequest","output_type":".remoteread.TargetList","options":{}}},{"http":{"Pattern":{"Post":"/targets/start/{target.meta.clusterId}/{target.meta.name}"},"body":"*"},"method":{"name":"Start","input_type":".remoteread.StartReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/targets/stop/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"Stop","input_type":".remoteread.StopReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status"},"body":"*"},"method":{"name":"GetTargetStatus","input_type":".remoteread.TargetStatusRequest","output_type":".remoteread.TargetStatus","options":{}}},{"http":{"Pattern":{"Post":"/edit"},"body":"*"},"method":{"name":"Discover","input_type":".remoteread.DiscoveryRequest","output_type":".remoteread.DiscoveryResponse","options":{}}}]} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service node.metrics.NodeConfiguration + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "alerting.ops.AlertingAdmin"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterConfiguration"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/ConfigureCluster"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterStatus"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /ready", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO plugin.opni.alerting starting opni embedded server {"addr": "127.0.0.1:43845"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PUT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PATCH /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "HEAD /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/InstallCluster"} + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "node.metrics.NodeConfiguration"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "DELETE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "DELETE /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/UninstallCluster"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "node.metrics.NodeConfiguration", "rules": 4} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/Info"} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "node.metrics.NodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".node.metrics.MetricsCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".node.metrics.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "alerting.ops.AlertingAdmin", "rules": 6} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "alerting.ops.AlertingAdmin", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".alerting.ops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".alerting.ops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/install"}},"method":{"name":"InstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".alerting.ops.UninstallRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/info"}},"method":{"name":"Info","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ComponentInfo","options":{}}}]} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service alerting.ops.ConfigReconciler + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.ops.ConfigReconciler/SyncConfig"} + 2023-08-15T01:40:56Z INFO mgmt service has no http rules {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "TRACE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service alerting.node.AlertingNodeConfiguration + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "alerting.node.AlertingNodeConfiguration"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "alerting.node.AlertingNodeConfiguration", "rules": 4} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "alerting.node.AlertingNodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".alerting.node.AlertingCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".alerting.node.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service alerting.node.NodeAlertingCapability + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "alerting.node.NodeAlertingCapability"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.node.NodeAlertingCapability/Sync"} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "alerting.node.NodeAlertingCapability", "rules": 1} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "alerting.node.NodeAlertingCapability", "rules": [{"http":{"Pattern":{"Post":"/v1/node/sync"},"body":"*"},"method":{"name":"Sync","input_type":".alerting.node.AlertingCapabilityConfig","output_type":".alerting.node.SyncResponse","options":{}}}]} + 2023-08-15T01:40:56Z INFO test.env Waiting for gateway to start... + 2023-08-15T01:40:56Z INFO mgmt management HTTP server starting {"address": "localhost:38617"} + 2023-08-15T01:40:56Z INFO test.env loaded 12 plugins + 2023-08-15T01:40:56Z INFO mgmt management gRPC server starting {"address": "127.0.0.1:35907"} + 2023-08-15T01:40:56Z INFO gateway.grpc gateway gRPC server starting {"address": "127.0.0.1:43731"} + 2023-08-15T01:40:56Z INFO gateway.http gateway HTTP server starting {"api": "127.0.0.1:42365", "metrics": "127.0.0.1:45663"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/tokens"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/tokens/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/watch"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/clusters/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/certs"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}/health"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/health/watch"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/clusters/{cluster.id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/roles"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/roles"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/roles/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/rolebindings"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/rolebindings"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/subjectaccess"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/apiextensions"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/config"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/config"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/capabilities"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/capabilities/{name}/installer"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/install"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/uninstall"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/status"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/status"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/cancel"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/dashboard/settings"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/dashboard/settings"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/all_user_stats"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/write_metrics"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules/{namespace}/{groupName}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/metadata"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/rules"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/CortexAdmin/rules/{groupName}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/flush_blocks"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/metadata"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/labels"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/status"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/config"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/raw"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/configuration"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/configure"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/status"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/uninstall"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertConditions/groups"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/configure"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list"} + 2023-08-15T01:40:56Z INFO test.env Gateway started + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertConditions/configure"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/choices"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/configure"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/status/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/withStatus"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/clone"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/silences"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/silences"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/timeline"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/list/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/toggleNotifications"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertEndpoints/list"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/delete"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/test"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/trigger"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/push"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/notifications/list"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/alarms/list"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/RemoteReadGateway/targets/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/start/{target.meta.clusterId}/{target.meta.name}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/stop/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/status"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/edit"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config/{node.id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/configuration"} + 2023-08-15T01:40:56Z DEBUG plugin.alerting-test-cluster-driver Syncer start : alerting-server --syncer.alertmanager.config.file=/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml --syncer.listen.address=:41087 --syncer.alertmanager.address=http://127.0.0.1:36163 --syncer.gateway.join.address=:35907 syncer {"config-file": "/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/configure"} + 2023-08-15T01:40:56Z INFO plugin.alerting-test-cluster-driver Starting AlertManager {"config-file": "/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml", "alertmanager-port": 36163, "opni-port": 33761} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/status"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/install"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/uninstall"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/info"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config/{node.id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/NodeAlertingCapability/v1/node/sync"} + ts=2023-08-15T01:40:57.061Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:40:57.061Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + ts=2023-08-15T01:40:57.064Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=46765 + ts=2023-08-15T01:40:57.065Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + ts=2023-08-15T01:40:57.096Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml + ts=2023-08-15T01:40:57.096Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml + ts=2023-08-15T01:40:57.099Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:36163 + ts=2023-08-15T01:40:57.099Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:36163 + 2023-08-15T01:40:57Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:57Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:57Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:57Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:57Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:57Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:57Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:57Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:57Z INFO plugin.alerting-test-cluster-driver Starting AlertManager Syncer {"config-file": "/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml", "port": 41087} + 2023-08-15T01:40:57Z INFO plugin.alerting-test-cluster-driver AlertManager started {"config-file": "/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml", "address": "http://127.0.0.1:36163"} + 2023-08-15T01:40:57Z INFO plugin.alerting updating known peers : [{http://127.0.0.1:36163 http://127.0.0.1:43845}] + 2023-08-15T01:40:57Z INFO test.env Waiting for cortex to start... + 2023-08-15T01:40:57Z INFO plugin.metrics.metrics-backend node sync requested {"cluster": "(all)", "capability": "metrics"} + 2023-08-15T01:40:57Z INFO test starting agent {"id": "agent1", "address": "127.0.0.1:36993", "version": "v2"} + 2023-08-15T01:40:57Z DEBUG agent using log level: debug + 2023-08-15T01:40:57Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:40:58Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent1 + 2023-08-15T01:40:58Z INFO agent bootstrap completed successfully + 2023-08-15T01:40:58Z WARN jetstream disconnected from jetstream + 2023-08-15T01:40:58Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:40:58Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent1 is now active + 2023-08-15T01:40:58Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.HTTPAPIExtension"]} + 2023-08-15T01:40:58Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.Health", "opni.apiextensions.StreamAPIExtension", "opni.backends.Capability"]} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:40:58Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent1 is now active + 2023-08-15T01:40:58Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:58Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:58Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:40:58Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:40:58Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:40:58Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:58Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:58Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:40:58Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:40:58Z INFO agent using unmanaged plugin loader + 2023-08-15T01:40:58Z INFO agent agent http server starting {"address": "127.0.0.1:36993"} + 2023-08-15T01:40:58Z INFO agent connecting to gateway... + 2023-08-15T01:40:58Z DEBUG gateway ratelimit: 50.000000 available + 2023-08-15T01:40:58Z DEBUG gateway-client authenticated {"id": "agent1"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream stream connected + 2023-08-15T01:40:58Z DEBUG gateway.grpc handling new stream connection + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent1"} + 2023-08-15T01:40:58Z DEBUG commands/alerting.go:72 syncer gateway join address:35907 + 2023-08-15T01:40:58Z INFO plugin.alerting-syncer syncer/syncer_server.go:90 starting alerting syncer server as identity d698ddbc-b7c2-4866-868b-f42f0bb9ece1 + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent1"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:108 acquired gateway alerting client + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:144 starting alerting syncer stream subscription + 2023-08-15T01:40:58Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "f381c452-12ec-4c96-b432-54636f604a96"} + 2023-08-15T01:40:58Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "b1d328c9-a063-4e6f-993d-10490e6e76fb"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:150 starting new stream subscription + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + ts=2023-08-15T01:40:58.098819998Z caller=experimental.go:19 level=warn msg="experimental feature in use" feature=tenant-federation + 2023-08-15T01:40:58Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml"} + ts=2023-08-15T01:40:58.106Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml + ts=2023-08-15T01:40:58.106Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml + ts=2023-08-15T01:40:58.108418462Z caller=modules.go:331 level=warn msg="Worker address is empty in single binary mode. Attempting automatic worker configuration. If queries are unresponsive consider configuring the worker explicitly." address=127.0.0.1:33573 + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:58Z DEBUG gateway.sync agent connected {"id": "agent1"} + 2023-08-15T01:40:58Z INFO gateway.monitor received status update {"id": "agent1", "connected": true} + 2023-08-15T01:40:58Z DEBUG gateway.delegate agent connected {"id": "agent1"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:58Z INFO agent gateway connected + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:40:58Z INFO gateway.monitor received health update {"id": "agent1", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:40:58Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:40:58Z"} + 2023-08-15T01:40:58Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:58Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:40:58Z INFO test.env Cortex started {"httpAddress": "https://localhost:46751", "grpcAddress": "localhost:33573"} + 2023-08-15T01:40:58Z INFO test starting agent {"id": "agent2", "address": "127.0.0.1:46475", "version": "v2"} + 2023-08-15T01:40:58Z DEBUG agent using log level: debug + 2023-08-15T01:40:58Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting rule sync is disabled {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:40:58Z INFO gateway.monitor received health update {"id": "agent1", "ready": true, "conditions": []} + 2023-08-15T01:40:58Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:40:58Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:40:58Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:40:58Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent1", "time": "2023-08-15T01:40:58Z"} + 2023-08-15T01:40:58Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:58Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:40:58Z DEBUG test.env configuring node {"node": "agent1", "driver": "otel"} + 2023-08-15T01:40:58Z DEBUG test.env configuring node {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:58Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "otel"} + 2023-08-15T01:40:58Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent1"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:40:58Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:58Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:58Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent2 + 2023-08-15T01:40:58Z INFO agent bootstrap completed successfully + 2023-08-15T01:40:58Z WARN jetstream disconnected from jetstream + 2023-08-15T01:40:58Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent2 is now active + 2023-08-15T01:40:58Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:40:58Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent2 is now active + 2023-08-15T01:40:58Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.HTTPAPIExtension"]} + 2023-08-15T01:40:58Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:40:58Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:58Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:58Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:40:58Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:40:58Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:40:58Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:58Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:58Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:40:58Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:40:58Z INFO agent using unmanaged plugin loader + 2023-08-15T01:40:58Z INFO agent agent http server starting {"address": "127.0.0.1:46475"} + 2023-08-15T01:40:58Z INFO agent connecting to gateway... + 2023-08-15T01:40:58Z DEBUG gateway ratelimit: 50.000000 available + 2023-08-15T01:40:58Z DEBUG gateway-client authenticated {"id": "agent2"} + 2023-08-15T01:40:58Z DEBUG gateway.grpc handling new stream connection + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent2"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream stream connected + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent2"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:40:58Z DEBUG gateway.sync agent connected {"id": "agent2"} + 2023-08-15T01:40:58Z INFO gateway.monitor received status update {"id": "agent2", "connected": true} + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:58Z DEBUG gateway.delegate agent connected {"id": "agent2"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:40:58Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:40:58Z INFO agent gateway connected + 2023-08-15T01:40:58Z INFO gateway.monitor received health update {"id": "agent2", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:40:58Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent2", "time": "2023-08-15T01:40:58Z"} + 2023-08-15T01:40:58Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:58Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:40:58Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent2"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:40:58Z DEBUG test.env configuring node {"node": "agent2", "driver": "otel"} + 2023-08-15T01:40:58Z DEBUG test.env nothing to do (already stopped) {"node": "agent2", "driver": "otel"} + 2023-08-15T01:40:58Z DEBUG test.env configuring node {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:40:58Z DEBUG test.env nothing to do (already stopped) {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:40:58Z INFO gateway.monitor received health update {"id": "agent2", "ready": true, "conditions": []} + 2023-08-15T01:40:58Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:58Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:40:58Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent2", "time": "2023-08-15T01:40:58Z"} + 2023-08-15T01:40:58Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:58Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting rule sync is disabled {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:58Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent1", "capabilities": ["metrics"]} + 2023-08-15T01:40:58Z DEBUG plugin.alerting ignoring sync request due to capability filter 'alerting' {"component": "health-cfg-sync"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics received sync request + 2023-08-15T01:40:58Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:40:58Z INFO plugin.metrics.metrics-backend node sync requested {"cluster": "agent1", "capability": "metrics"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent1", "time": "2023-08-15T01:40:58Z"} + 2023-08-15T01:40:58Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:40:58Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent1"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics starting rule sync + 2023-08-15T01:40:58Z DEBUG test.env configuring node {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics enabling http server + 2023-08-15T01:40:58Z INFO test.env starting prometheus {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics configuring rule discovery + 2023-08-15T01:40:58Z DEBUG plugin.metrics rule discovery configured + 2023-08-15T01:40:58Z DEBUG plugin.metrics rule discovery notifier configured {"interval": "15m0s"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics starting rule group update notifier + 2023-08-15T01:40:58Z DEBUG test.env configuring node {"node": "agent1", "driver": "otel"} + 2023-08-15T01:40:58Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "otel"} + STEP: verifying the only the default group exists @ 08/15/23 01:40:58.484 + 2023-08-15T01:40:58Z INFO test.env Waiting for prometheus to start... + 2023-08-15T01:40:58Z DEBUG plugin.alerting Updating alert condition id:"bGpeDG5QAFzsEZB5R9j8kk" {"component": "alarms", "handler": "UpdateAlertCondition"} + STEP: veriying the condition in the old group storage is not found @ 08/15/23 01:40:58.504 + STEP: veryfing the condition has been move to the new group storage @ 08/15/23 01:40:58.504 +• [1.697 seconds] +------------------------------ +Invalidated and clean up suite test when we delete an alarm of the metrics type should successfully submit the alarm for deletion [integration] +/home/ec2-user/opni/test/plugins/alerting/delete_test.go:199 + STEP: verifying the metrics conditions eventually are active successfully @ 08/15/23 01:40:58.511 + 2023/08/15 01:40:58 http: TLS handshake error from 127.0.0.1:40808: EOF + 2023-08-15T01:40:58Z INFO test.env Prometheus started {"address": "http://localhost:44699", "dir": "/tmp/opni-test-522595029/prometheus/agent1", "agentId": "agent1"} + 2023-08-15T01:40:58Z INFO test.env started prometheus {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:58Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:58Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:58Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:58Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:58Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "6af872f48e9930297eb9b7408690ad0524d9893461a72a1bd61fde0594914d2d"} + 2023-08-15T01:40:58Z INFO plugin.alerting syncing 6 conditions {"component": "alarms"} + 2023-08-15T01:40:58Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (6af872f48e9930297eb9b7408690ad0524d9893461a72a1bd61fde0594914d2d) config message + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml"} + ts=2023-08-15T01:40:58.890Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml + ts=2023-08-15T01:40:58.891Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting received condition update: name:"agent disconnect" description:"some disconnect to remove" alertType:{system:{clusterId:{id:"agent1"} timeout:{seconds:600}}} silence:{} lastUpdated:{seconds:1692063658 nanos:493414883} id:"bGpeDG5QAFzsEZB5R9j8kk" metadata:{key:"opni.io/alarm-inactive" value:"true"} groupId:"new-group" {"component": "alarms", "onSystemConditionCreate": "bGpeDG5QAFzsEZB5R9j8kk"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting Creating agent disconnect with timeout 10m0s {"component": "alarms", "onSystemConditionCreate": "bGpeDG5QAFzsEZB5R9j8kk"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting name: cf9Lm3PynGemtUxhLZ2Ezk-opni-alerting + rules: + - alert: cf9Lm3PynGemtUxhLZ2Ezk + expr: opni:custom:prometheus_query{opni_uuid="cf9Lm3PynGemtUxhLZ2Ezk"} + for: 1s + labels: + opni_promql: cf9Lm3PynGemtUxhLZ2Ezk + opni_severity: Info + opni_uuid: cf9Lm3PynGemtUxhLZ2Ezk + annotations: + OpniAlarmName: test + OpniHeader: test + OpniSummary: Prometheus query + opni_clusterId: agent1 + opni_fingerprint: '{{ "ALERTS_FOR_STATE{opni_uuid=\"cf9Lm3PynGemtUxhLZ2Ezk\"} OR vector(0)" | query | first | value | printf "%.0f" }}' + opni_goldenSignal: Custom + opni_uuid: cf9Lm3PynGemtUxhLZ2Ezk + - record: opni:custom:prometheus_query + expr: sum(up > 0) > 0 + labels: + opni_uuid: cf9Lm3PynGemtUxhLZ2Ezk + {"component": "alarms", "Expr": "user-query"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting received condition update: name:"agent-disconnect" description:"Alert when the downstream agent disconnects from the opni upstream" labels:"agent-disconnect" labels:"opni" labels:"_default" severity:Critical alertType:{system:{clusterId:{id:"agent1"} timeout:{seconds:1}}} lastUpdated:{seconds:1692063658 nanos:28040087} id:"dCfr8i4Q6EHPvxZwW9gLt6" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onSystemConditionCreate": "dCfr8i4Q6EHPvxZwW9gLt6"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting Creating agent disconnect with timeout 1s {"component": "alarms", "onSystemConditionCreate": "dCfr8i4Q6EHPvxZwW9gLt6"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting received condition update: name:"agent-disconnect" description:"Alert when the downstream agent disconnects from the opni upstream" labels:"agent-disconnect" labels:"opni" labels:"_default" severity:Critical alertType:{system:{clusterId:{id:"agent2"} timeout:{seconds:1}}} lastUpdated:{seconds:1692063658 nanos:297438417} id:"34e6zCJWGxLSxNSP97cftd" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting Creating agent disconnect with timeout 1s {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting received condition update: name:"agent-capability-unhealthy" description:"Alert when some downstream agent capability becomes unhealthy" labels:"agent-capability-health" labels:"opni" labels:"_default" severity:Critical alertType:{downstreamCapability:{clusterId:{id:"agent2"} capabilityState:"Failure" capabilityState:"Pending" for:{seconds:1}}} lastUpdated:{seconds:1692063658 nanos:302402760} id:"stdRzQww4KPXDy5r3dVAh6" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onCapabilityStatusCreate": "stdRzQww4KPXDy5r3dVAh6"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting received condition update: name:"agent-capability-unhealthy" description:"Alert when some downstream agent capability becomes unhealthy" labels:"agent-capability-health" labels:"opni" labels:"_default" severity:Critical alertType:{downstreamCapability:{clusterId:{id:"agent1"} capabilityState:"Failure" capabilityState:"Pending" for:{seconds:1}}} lastUpdated:{seconds:1692063658 nanos:54065227} id:"FsECjbp6G7xntHLX5o9DjR" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onCapabilityStatusCreate": "FsECjbp6G7xntHLX5o9DjR"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting Creating agent capability unhealthy with timeout 1s {"component": "alarms", "onCapabilityStatusCreate": "stdRzQww4KPXDy5r3dVAh6"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting Creating agent capability unhealthy with timeout 1s {"component": "alarms", "onCapabilityStatusCreate": "FsECjbp6G7xntHLX5o9DjR"} + 2023-08-15T01:40:58Z INFO plugin.alerting successfully synced (6/6) conditions {"component": "alarms"} + 2023-08-15T01:40:58Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40826: EOF + ts=2023-08-15T01:40:59.066Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000373264s + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40830: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40840: read tcp 127.0.0.1:46751->127.0.0.1:40840: read: connection reset by peer + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40846: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40858: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40864: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40872: read tcp 127.0.0.1:46751->127.0.0.1:40872: read: connection reset by peer + STEP: deleting the metrics condition @ 08/15/23 01:40:59.139 + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40878: EOF +• [0.645 seconds] +------------------------------ +Invalidated and clean up suite test when we delete an alarm of the metrics type should clean up the dependencies of this alarm [integration] +/home/ec2-user/opni/test/plugins/alerting/delete_test.go:258 + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40888: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40904: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40916: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40930: EOF + 2023-08-15T01:40:59Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:59Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:59Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:59Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + ts=2023-08-15T01:40:59.721968567Z caller=loader.go:117 level=warn msg="bucket index not found" user=agent1 + 2023-08-15T01:40:59Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:59Z INFO plugin.alerting syncing 6 conditions {"component": "alarms"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "6af872f48e9930297eb9b7408690ad0524d9893461a72a1bd61fde0594914d2d"} + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40938: read tcp 127.0.0.1:46751->127.0.0.1:40938: read: connection reset by peer + 2023-08-15T01:40:59Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (6af872f48e9930297eb9b7408690ad0524d9893461a72a1bd61fde0594914d2d) config message + 2023-08-15T01:40:59Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40948: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40950: read tcp 127.0.0.1:46751->127.0.0.1:40950: read: connection reset by peer + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40956: read tcp 127.0.0.1:46751->127.0.0.1:40956: read: connection reset by peer + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40968: read tcp 127.0.0.1:46751->127.0.0.1:40968: read: connection reset by peer + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:41012: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40996: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40984: read tcp 127.0.0.1:46751->127.0.0.1:40984: read: connection reset by peer + 2023-08-15T01:40:59Z INFO plugin.alerting successfully synced (6/6) conditions {"component": "alarms"} + 2023-08-15T01:40:59Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063658 nanos:356502993 {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063658 nanos:119253770 {"component": "alarms", "onSystemConditionCreate": "bGpeDG5QAFzsEZB5R9j8kk"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063658 nanos:119253770 {"component": "alarms", "onSystemConditionCreate": "bGpeDG5QAFzsEZB5R9j8kk"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063658 nanos:356502993 {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063658 nanos:119253770 {"component": "alarms", "onSystemConditionCreate": "dCfr8i4Q6EHPvxZwW9gLt6"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063658 nanos:356502993 {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063658 nanos:119253770 {"component": "alarms", "onSystemConditionCreate": "dCfr8i4Q6EHPvxZwW9gLt6"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063658 nanos:119253770 {"component": "alarms", "onSystemConditionCreate": "bGpeDG5QAFzsEZB5R9j8kk"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063658 nanos:119253770 {"component": "alarms", "onSystemConditionCreate": "dCfr8i4Q6EHPvxZwW9gLt6"} +• [0.974 seconds] +------------------------------ +Invalidated and clean up suite test when we delete an alarm of the metrics type should clean up the alarm configuration itself [integration] +/home/ec2-user/opni/test/plugins/alerting/delete_test.go:273 +• [0.004 seconds] +------------------------------ +Invalidated and clean up suite test when We uninstall metrics capability from a cluster that has metrics alarms should switch these alarms to the invalidated state [integration] +/home/ec2-user/opni/test/plugins/alerting/delete_test.go:293 + 2023-08-15T01:41:00Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent1", "capabilities": ["metrics"]} + 2023-08-15T01:41:00Z DEBUG plugin.alerting ignoring sync request due to capability filter 'alerting' {"component": "health-cfg-sync"} + 2023-08-15T01:41:00Z DEBUG plugin.metrics received sync request + 2023-08-15T01:41:00Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:41:00Z INFO plugin.metrics.metrics-backend node sync requested {"cluster": "agent1", "capability": "metrics"} + 2023-08-15T01:41:00Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent1", "time": "2023-08-15T01:41:00Z"} + 2023-08-15T01:41:00Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:41:00Z DEBUG test.env configuring node {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:41:00Z INFO test.env stopping prometheus {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:41:00Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent1"} + 2023-08-15T01:41:00Z DEBUG plugin.metrics stopping rule sync + 2023-08-15T01:41:00Z DEBUG plugin.metrics disabling http server + 2023-08-15T01:41:00Z WARN plugin.metrics rule discovery stream closing {"error": "context canceled"} + 2023-08-15T01:41:00Z DEBUG test.env configuring node {"node": "agent1", "driver": "otel"} + 2023-08-15T01:41:00Z DEBUG plugin.metrics rule discovery stream closing {"error": "context canceled"} + 2023-08-15T01:41:00Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "otel"} + 2023-08-15T01:41:00Z INFO tasks Stored data will not be deleted {"id": "agent1"} + 2023-08-15T01:41:00Z INFO tasks Uninstalling metrics capability for this cluster {"id": "agent1"} + 2023-08-15T01:41:00Z INFO tasks Time series data will not be deleted {"id": "agent1"} +• [0.034 seconds] +------------------------------ + 2023-08-15T01:41:00Z INFO tasks Removing capability from cluster metadata {"id": "agent1"} +Invalidated and clean up suite test when we delete an alarm of the internal type should successfully submit the alarm for deletion [integration] +/home/ec2-user/opni/test/plugins/alerting/delete_test.go:327 +• [0.005 seconds] +------------------------------ +Invalidated and clean up suite test when we delete an alarm of the internal type should clean up the alarm configuration itself [integration] +/home/ec2-user/opni/test/plugins/alerting/delete_test.go:334 + 2023-08-15T01:41:00Z INFO tasks Capability uninstalled successfully {"id": "agent1"} + 2023-08-15T01:41:00Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:00Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:00Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:00Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:00Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:00Z INFO plugin.alerting syncing 5 conditions {"component": "alarms"} + 2023-08-15T01:41:00Z INFO plugin.alerting evaluation context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "bGpeDG5QAFzsEZB5R9j8kk"} + 2023-08-15T01:41:00Z INFO plugin.alerting evaluation context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "bGpeDG5QAFzsEZB5R9j8kk"} + 2023-08-15T01:41:00Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:41:00Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "dCfr8i4Q6EHPvxZwW9gLt6"} + 2023-08-15T01:41:00Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:41:00Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "b1d328c9-a063-4e6f-993d-10490e6e76fb"} + 2023-08-15T01:41:00Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "dCfr8i4Q6EHPvxZwW9gLt6"} + 2023-08-15T01:41:00Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "f381c452-12ec-4c96-b432-54636f604a96"} + 2023-08-15T01:41:00Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onCapabilityStatusCreate": "stdRzQww4KPXDy5r3dVAh6"} + 2023-08-15T01:41:00Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onCapabilityStatusCreate": "FsECjbp6G7xntHLX5o9DjR"} + 2023-08-15T01:41:00Z INFO plugin.alerting shutting down cluster driver update handler + 2023-08-15T01:41:00Z DEBUG plugin.alerting closing cortex cluster status watcher... {"watcher": "cortex-cluster-status"} + 2023-08-15T01:41:00Z DEBUG gateway.sync agent disconnected {"id": "agent2"} + 2023-08-15T01:41:00Z DEBUG gateway.delegate agent disconnected {"id": "agent2"} + 2023-08-15T01:41:00Z INFO gateway.grpc agent stream closing {"error": "context canceled"} + 2023-08-15T01:41:00Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:00Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:00Z INFO gateway shutting down plugins + 2023-08-15T01:41:00Z WARN plugin.alerting failed to receive cluster health status from grpc stream, retrying... + ts=2023-08-15T01:41:00.881Z caller=alertmanager_main.go:601 level=info msg="Received SIGTERM, exiting gracefully..." + ts=2023-08-15T01:41:00.881Z caller=cluster.go:690 level=info component=cluster msg="gossip not settled but continuing anyway" polls=1 elapsed=3.815512384s + 2023-08-15T01:41:00Z DEBUG plugin.alerting received condition update: name:"agent-disconnect" description:"Alert when the downstream agent disconnects from the opni upstream" labels:"agent-disconnect" labels:"opni" labels:"_default" severity:Critical alertType:{system:{clusterId:{id:"agent2"} timeout:{seconds:1}}} lastUpdated:{seconds:1692063658 nanos:297438417} id:"34e6zCJWGxLSxNSP97cftd" metadata:{key:"opni.io/alarm-hash" value:"4029a2c21ef580bbc9f2eaf1add48fb7caadc895c9af7002c2cb5678775bb7a0"} {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:41:00Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error + 2023-08-15T01:41:00Z DEBUG gateway.sync agent disconnected {"id": "agent1"} + 2023-08-15T01:41:00Z INFO gateway.grpc agent stream closing {"error": "context canceled"} + 2023-08-15T01:41:00Z DEBUG gateway.delegate agent disconnected {"id": "agent1"} + 2023-08-15T01:41:00Z DEBUG plugin.alerting Creating agent disconnect with timeout 1s {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:41:00Z WARN jetstream disconnected from jetstream +write tcp 127.0.0.1:49356->127.0.0.1:42403: write: connection reset by peer on connection [37] + 2023-08-15T01:41:00Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:00Z INFO gateway all plugins shut down + 2023-08-15T01:41:00Z DEBUG plugin.alerting received condition update: name:"agent-capability-unhealthy" description:"Alert when some downstream agent capability becomes unhealthy" labels:"agent-capability-health" labels:"opni" labels:"_default" severity:Critical alertType:{downstreamCapability:{clusterId:{id:"agent2"} capabilityState:"Failure" capabilityState:"Pending" for:{seconds:1}}} lastUpdated:{seconds:1692063658 nanos:302402760} id:"stdRzQww4KPXDy5r3dVAh6" metadata:{key:"opni.io/alarm-hash" value:"5f135eda476394945cbeaf43e3b102d3cbf5a0861e357ae3342ee6fcb86f5d55"} {"component": "alarms", "onCapabilityStatusCreate": "stdRzQww4KPXDy5r3dVAh6"} + 2023-08-15T01:41:00Z DEBUG plugin.alerting Creating agent capability unhealthy with timeout 1s {"component": "alarms", "onCapabilityStatusCreate": "stdRzQww4KPXDy5r3dVAh6"} +write tcp 127.0.0.1:49356->127.0.0.1:42403: write: broken pipe on connection [37] + 2023-08-15T01:41:00Z WARN gateway http server exited with error {"error": "context canceled"} + 2023-08-15T01:41:00Z WARN test.env gateway server exited with error {"error": "context canceled"} + 2023-08-15T01:41:00Z WARN plugin.alerting-syncer syncer/syncer_server.go:163 remote syncer unavailable, reconnecting, ... + 2023-08-15T01:41:00Z WARN test.env management server exited with error {"error": "context canceled"} + 2023-08-15T01:41:00Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "dada5d033f242ec39f88a12fdcc651db58bcfd009f2f1e5aa740ac523f4034bd"} + 2023-08-15T01:41:00Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:00Z ERROR plugin.alerting EOF + 2023-08-15T01:41:00Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:00Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:00Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:41:00Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:00Z INFO test.env Cortex stopping... + 2023-08-15T01:41:00Z ERROR plugin.alerting failed to receive cluster event : rpc error: code = Unavailable desc = error reading from server: read unix @->/tmp/plugin1670833980: use of closed network connection + 2023-08-15T01:41:00Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:00Z ERROR test agent "agent2" exited: context canceled + 2023-08-15T01:41:00Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:00Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:41:00Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:00Z ERROR test agent "agent1" exited: context canceled + 2023-08-15T01:41:00Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} +• [0.721 seconds] +------------------------------ +Node Config should initially have all nodes using the default config [integration] +/home/ec2-user/opni/test/plugins/alerting/node_config_test.go:122 + 2023-08-15T01:41:00Z INFO test.env Starting test environment + 2023-08-15T01:41:00Z INFO test.env Copying 7 files from embedded testdata/cortex to /tmp/opni-test-4032888549/cortex + 2023-08-15T01:41:00Z INFO test.env Waiting for jetstream to start... + 2023-08-15T01:41:00Z INFO test.env Cortex stopped + [231696] 2023/08/15 01:41:00.905057 [INF] Starting nats-server + [231696] 2023/08/15 01:41:00.905101 [INF] Version: 2.9.21 + [231696] 2023/08/15 01:41:00.905106 [INF] Git: [b2e7725] + [231696] 2023/08/15 01:41:00.905109 [INF] Name: NDUJMJCTVOXVPMTYVHH2W4VPQT6Q42NVYV2OALV44OCWIHI25NDRXJA3 + [231696] 2023/08/15 01:41:00.905115 [INF] Node: W4euMW8u + [231696] 2023/08/15 01:41:00.905119 [INF] ID: NDUJMJCTVOXVPMTYVHH2W4VPQT6Q42NVYV2OALV44OCWIHI25NDRXJA3 + [231696] 2023/08/15 01:41:00.905126 [INF] Using configuration file: /tmp/opni-test-4032888549/jetstream/jetstream.conf + [231696] 2023/08/15 01:41:00.905621 [INF] Starting JetStream + [231696] 2023/08/15 01:41:00.905794 [INF] _ ___ _____ ___ _____ ___ ___ _ __ __ + [231696] 2023/08/15 01:41:00.905808 [INF] _ | | __|_ _/ __|_ _| _ \ __| /_\ | \/ | + [231696] 2023/08/15 01:41:00.905812 [INF] | || | _| | | \__ \ | | | / _| / _ \| |\/| | + [231696] 2023/08/15 01:41:00.905815 [INF] \__/|___| |_| |___/ |_| |_|_\___/_/ \_\_| |_| + [231696] 2023/08/15 01:41:00.905818 [INF] + [231696] 2023/08/15 01:41:00.905822 [INF] https://docs.nats.io/jetstream + [231696] 2023/08/15 01:41:00.905825 [INF] + [231696] 2023/08/15 01:41:00.905828 [INF] ---------------- JETSTREAM ---------------- + [231696] 2023/08/15 01:41:00.905833 [INF] Max Memory: 11.71 GB + [231696] 2023/08/15 01:41:00.905837 [INF] Max Storage: 5.29 GB + [231696] 2023/08/15 01:41:00.905841 [INF] Store Directory: "/tmp/opni-test-4032888549/jetstream/data/jetstream" + [231696] 2023/08/15 01:41:00.905843 [INF] ------------------------------------------- + [231696] 2023/08/15 01:41:00.906224 [INF] Listening for client connections on 0.0.0.0:44491 + [231696] 2023/08/15 01:41:00.906543 [INF] Server is ready + 2023-08-15T01:41:00Z INFO test.env Jetstream started + 2023-08-15T01:41:00Z DEBUG keyring loaded ephemeral key {"path": "/tmp/opni-test-4032888549/keyring/local-agent.json", "usage": "auth", "labels": {"opni.io/session-attribute":"local"}} + 2023-08-15T01:41:00Z DEBUG gateway found 0 plugins + 2023-08-15T01:41:00Z DEBUG gateway loaded plugin manifest {"plugins": 0} + 2023-08-15T01:41:00Z INFO gateway.cache compressing and archiving plugins... + 2023-08-15T01:41:00Z DEBUG gateway.cache added 0 new plugins to cache + 2023-08-15T01:41:00Z INFO gateway running plugin cache gc + 2023-08-15T01:41:00Z INFO gateway.update-server registering update handler for strategy "binary" + 2023-08-15T01:41:00Z INFO gateway.update-server registering update handler for strategy "kubernetes" + 2023-08-15T01:41:00Z INFO gateway.update-server registering update handler for strategy "noop" + 2023-08-15T01:41:00Z DEBUG gateway.grpc registering service {"service": "control.HealthListener"} + 2023-08-15T01:41:00Z DEBUG gateway.grpc registering internal service {"service": "stream.Delegate"} + 2023-08-15T01:41:00Z DEBUG gateway.grpc registering service {"service": "core.Pinger"} + 2023-08-15T01:41:00Z INFO test.env Loading gateway plugins... + 2023-08-15T01:41:00Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:00Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:00Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.System", "opni.apiextensions.HTTPAPIExtension", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.ManagementAPIExtension", "opni.backends.Capability", "opni.Metrics"]} + 2023-08-15T01:41:00Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.System", "opni.apiextensions.HTTPAPIExtension", "opni.apiextensions.ManagementAPIExtension", "opni.Metrics", "opni.backends.Capability", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Metrics"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Metrics"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:00Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/metrics", "capability": "metrics"} + 2023-08-15T01:41:00Z INFO mgmt got extension descriptor for service cortexadmin.CortexAdmin + 2023-08-15T01:41:00Z INFO mgmt got extension descriptor for service alerting.AlertConditions + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.System"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:00Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_alerting"} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:00Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/alerting", "capability": "alerting"} + 2023-08-15T01:41:00Z DEBUG gateway.grpc plugin rejected internal stream connection {"plugin": "plugin_alerting"} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:00Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_metrics"} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.System"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:00Z INFO mgmt loading service {"name": "cortexadmin.CortexAdmin"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/AllUserStats"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/WriteMetrics"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/Query"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/QueryRange"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetRule"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricMetadata"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ListRules"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/LoadRules"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/DeleteRule"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/FlushBlocks"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetSeriesMetrics"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricLabelSets"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexStatus"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexConfig"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ExtractRawSeries"} + 2023-08-15T01:41:00Z INFO mgmt loading http rules {"name": "cortexadmin.CortexAdmin", "rules": 17} + 2023-08-15T01:41:00Z DEBUG mgmt rule descriptors {"name": "cortexadmin.CortexAdmin", "rules": [{"http":{"Pattern":{"Get":"/all_user_stats"}},"method":{"name":"AllUserStats","input_type":".google.protobuf.Empty","output_type":".cortexadmin.UserIDStatsList","options":{}}},{"http":{"Pattern":{"Post":"/write_metrics"},"body":"*"},"method":{"name":"WriteMetrics","input_type":".cortexadmin.WriteRequest","output_type":".cortexadmin.WriteResponse","options":{}}},{"http":{"Pattern":{"Get":"/query"},"additional_bindings":[{"Pattern":{"Post":"/query"},"body":"*"}]},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query"},"body":"*"},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/query_range"},"additional_bindings":[{"Pattern":{"Post":"/query_range"},"body":"*"}]},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query_range"},"body":"*"},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/rules/{namespace}/{groupName}"}},"method":{"name":"GetRule","input_type":".cortexadmin.GetRuleRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/metadata"}},"method":{"name":"GetMetricMetadata","input_type":".cortexadmin.MetricMetadataRequest","output_type":".cortexadmin.MetricMetadata","options":{}}},{"http":{"Pattern":{"Get":"/rules"}},"method":{"name":"ListRules","input_type":".cortexadmin.ListRulesRequest","output_type":".cortexadmin.ListRulesResponse","options":{}}},{"http":{"Pattern":{"Post":"/rules"}},"method":{"name":"LoadRules","input_type":".cortexadmin.LoadRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/rules/{groupName}"}},"method":{"name":"DeleteRule","input_type":".cortexadmin.DeleteRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/flush_blocks"}},"method":{"name":"FlushBlocks","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/series/metadata"}},"method":{"name":"GetSeriesMetrics","input_type":".cortexadmin.SeriesRequest","output_type":".cortexadmin.SeriesInfoList","options":{}}},{"http":{"Pattern":{"Get":"/series/labels"}},"method":{"name":"GetMetricLabelSets","input_type":".cortexadmin.LabelRequest","output_type":".cortexadmin.MetricLabels","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetCortexStatus","input_type":".google.protobuf.Empty","output_type":".cortexadmin.CortexStatus","options":{}}},{"http":{"Pattern":{"Get":"/config"}},"method":{"name":"GetCortexConfig","input_type":".cortexadmin.ConfigRequest","output_type":".cortexadmin.ConfigResponse","options":{}}},{"http":{"Pattern":{"Get":"/series/raw"}},"method":{"name":"ExtractRawSeries","input_type":".cortexadmin.MatcherRequest","output_type":".cortexadmin.QueryResponse","options":{}}}]} + 2023-08-15T01:41:00Z INFO mgmt got extension descriptor for service cortexops.CortexOps + 2023-08-15T01:41:00Z DEBUG plugin.metrics/pkg/gateway.stream internal gateway stream connected + 2023-08-15T01:41:00Z INFO mgmt loading service {"name": "alerting.AlertConditions"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionGroups"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CreateAlertCondition"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/GetAlertCondition"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditions"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/UpdateAlertCondition"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionChoices"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeleteAlertCondition"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/AlertConditionStatus"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionsWithStatus"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CloneTo"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ActivateSilence"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeactivateSilence"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/Timeline"} + 2023-08-15T01:41:00Z INFO mgmt loading http rules {"name": "alerting.AlertConditions", "rules": 13} + 2023-08-15T01:41:01Z DEBUG mgmt rule descriptors {"name": "alerting.AlertConditions", "rules": [{"http":{"Pattern":{"Get":"/groups"}},"method":{"name":"ListAlertConditionGroups","input_type":".google.protobuf.Empty","output_type":".core.ReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertCondition","input_type":".alerting.AlertCondition","output_type":".alerting.ConditionReference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertCondition","input_type":".alerting.ConditionReference","output_type":".alerting.AlertCondition","options":{}}},{"http":{"Pattern":{"Post":"/list"},"body":"*"},"method":{"name":"ListAlertConditions","input_type":".alerting.ListAlertConditionRequest","output_type":".alerting.AlertConditionList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertCondition","input_type":".alerting.UpdateAlertConditionRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/choices"}},"method":{"name":"ListAlertConditionChoices","input_type":".alerting.AlertDetailChoicesRequest","output_type":".alerting.ListAlertTypeDetails","options":{}}},{"http":{"Pattern":{"Delete":"/configure"}},"method":{"name":"DeleteAlertCondition","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status/{id}"}},"method":{"name":"AlertConditionStatus","input_type":".alerting.ConditionReference","output_type":".alerting.AlertStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/list/withStatus"},"body":"*"},"method":{"name":"ListAlertConditionsWithStatus","input_type":".alerting.ListStatusRequest","output_type":".alerting.ListStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/clone"},"body":"*"},"method":{"name":"CloneTo","input_type":".alerting.CloneToRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/silences"},"body":"*"},"method":{"name":"ActivateSilence","input_type":".alerting.SilenceRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/silences"}},"method":{"name":"DeactivateSilence","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/timeline"}},"method":{"name":"Timeline","input_type":".alerting.TimelineRequest","output_type":".alerting.TimelineResponse","options":{}}}]} + 2023-08-15T01:41:01Z INFO mgmt got extension descriptor for service alerting.AlertEndpoints + 2023-08-15T01:41:01Z INFO mgmt loading service {"name": "alerting.AlertEndpoints"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/CreateAlertEndpoint"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/GetAlertEndpoint"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ToggleNotifications"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ListAlertEndpoints"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/UpdateAlertEndpoint"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/DeleteAlertEndpoint"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/TestAlertEndpoint"} + 2023-08-15T01:41:01Z INFO mgmt loading http rules {"name": "alerting.AlertEndpoints", "rules": 7} + 2023-08-15T01:41:01Z DEBUG mgmt rule descriptors {"name": "alerting.AlertEndpoints", "rules": [{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertEndpoint","input_type":".alerting.AlertEndpoint","output_type":".core.Reference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertEndpoint","input_type":".core.Reference","output_type":".alerting.AlertEndpoint","options":{}}},{"http":{"Pattern":{"Post":"/toggleNotifications"},"body":"*"},"method":{"name":"ToggleNotifications","input_type":".alerting.ToggleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/list"}},"method":{"name":"ListAlertEndpoints","input_type":".alerting.ListAlertEndpointsRequest","output_type":".alerting.AlertEndpointList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertEndpoint","input_type":".alerting.UpdateAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/delete"},"body":"*"},"method":{"name":"DeleteAlertEndpoint","input_type":".alerting.DeleteAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/test"},"body":"*"},"method":{"name":"TestAlertEndpoint","input_type":".alerting.TestAlertEndpointRequest","output_type":".alerting.TestAlertEndpointResponse","options":{}}}]} + 2023-08-15T01:41:01Z INFO mgmt got extension descriptor for service alerting.AlertNotifications + 2023-08-15T01:41:01Z INFO mgmt loading service {"name": "alerting.AlertNotifications"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/TriggerAlerts"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ResolveAlerts"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/PushNotification"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListNotifications"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListAlarmMessages"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListRoutingRelationships"} + 2023-08-15T01:41:01Z INFO mgmt loading http rules {"name": "alerting.AlertNotifications", "rules": 4} + 2023-08-15T01:41:01Z DEBUG mgmt rule descriptors {"name": "alerting.AlertNotifications", "rules": [{"http":{"Pattern":{"Post":"/trigger"},"body":"*"},"method":{"name":"TriggerAlerts","input_type":".alerting.TriggerAlertsRequest","output_type":".alerting.TriggerAlertsResponse","options":{}}},{"http":{"Pattern":{"Post":"/push"},"body":"*"},"method":{"name":"PushNotification","input_type":".alerting.Notification","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/notifications/list"},"body":"*"},"method":{"name":"ListNotifications","input_type":".alerting.ListNotificationRequest","output_type":".alerting.ListMessageResponse","options":{}}},{"http":{"Pattern":{"Post":"/alarms/list"},"body":"*"},"method":{"name":"ListAlarmMessages","input_type":".alerting.ListAlarmMessageRequest","output_type":".alerting.ListMessageResponse","options":{}}}]} + 2023-08-15T01:41:01Z INFO mgmt got extension descriptor for service alerting.ops.AlertingAdmin + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z INFO mgmt loading service {"name": "cortexops.CortexOps"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/cortexops.CortexOps/ConfigureCluster"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterStatus"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/cortexops.CortexOps/UninstallCluster"} + 2023-08-15T01:41:01Z INFO mgmt loading http rules {"name": "cortexops.CortexOps", "rules": 4} + 2023-08-15T01:41:01Z DEBUG mgmt rule descriptors {"name": "cortexops.CortexOps", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".cortexops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".cortexops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".cortexops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:41:01Z INFO mgmt got extension descriptor for service remoteread.RemoteReadGateway + 2023-08-15T01:41:01Z INFO mgmt loading service {"name": "alerting.ops.AlertingAdmin"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterConfiguration"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/trace", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/profile", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/block", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/heap", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/ready", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/healthy", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/ConfigureCluster"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterStatus"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/InstallCluster"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/UninstallCluster"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/Info"} + 2023-08-15T01:41:01Z INFO mgmt loading http rules {"name": "alerting.ops.AlertingAdmin", "rules": 6} + 2023-08-15T01:41:01Z DEBUG mgmt rule descriptors {"name": "alerting.ops.AlertingAdmin", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".alerting.ops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".alerting.ops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/install"}},"method":{"name":"InstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".alerting.ops.UninstallRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/info"}},"method":{"name":"Info","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ComponentInfo","options":{}}}]} + 2023-08-15T01:41:01Z INFO mgmt got extension descriptor for service alerting.ops.ConfigReconciler + 2023-08-15T01:41:01Z INFO mgmt loading service {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.ops.ConfigReconciler/SyncConfig"} + 2023-08-15T01:41:01Z INFO mgmt service has no http rules {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:41:01Z INFO mgmt got extension descriptor for service alerting.node.AlertingNodeConfiguration + 2023-08-15T01:41:01Z INFO mgmt loading service {"name": "remoteread.RemoteReadGateway"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/AddTarget"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/EditTarget"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/RemoveTarget"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/ListTargets"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Start"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Stop"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/GetTargetStatus"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Discover"} + 2023-08-15T01:41:01Z INFO mgmt loading http rules {"name": "remoteread.RemoteReadGateway", "rules": 8} + 2023-08-15T01:41:01Z DEBUG mgmt rule descriptors {"name": "remoteread.RemoteReadGateway", "rules": [{"http":{"Pattern":{"Post":"/targets"},"body":"*"},"method":{"name":"AddTarget","input_type":".remoteread.TargetAddRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Put":"/targets"},"body":"*"},"method":{"name":"EditTarget","input_type":".remoteread.TargetEditRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/targets/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"RemoveTarget","input_type":".remoteread.TargetRemoveRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/targets"}},"method":{"name":"ListTargets","input_type":".remoteread.TargetListRequest","output_type":".remoteread.TargetList","options":{}}},{"http":{"Pattern":{"Post":"/targets/start/{target.meta.clusterId}/{target.meta.name}"},"body":"*"},"method":{"name":"Start","input_type":".remoteread.StartReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/targets/stop/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"Stop","input_type":".remoteread.StopReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status"},"body":"*"},"method":{"name":"GetTargetStatus","input_type":".remoteread.TargetStatusRequest","output_type":".remoteread.TargetStatus","options":{}}},{"http":{"Pattern":{"Post":"/edit"},"body":"*"},"method":{"name":"Discover","input_type":".remoteread.DiscoveryRequest","output_type":".remoteread.DiscoveryResponse","options":{}}}]} + 2023-08-15T01:41:01Z INFO mgmt got extension descriptor for service node.metrics.NodeConfiguration + 2023-08-15T01:41:01Z INFO mgmt loading service {"name": "alerting.node.AlertingNodeConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading http rules {"name": "alerting.node.AlertingNodeConfiguration", "rules": 4} + 2023-08-15T01:41:01Z DEBUG mgmt rule descriptors {"name": "alerting.node.AlertingNodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".alerting.node.AlertingCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".alerting.node.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:41:01Z INFO mgmt got extension descriptor for service alerting.node.NodeAlertingCapability + 2023-08-15T01:41:01Z INFO mgmt loading service {"name": "alerting.node.NodeAlertingCapability"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.node.NodeAlertingCapability/Sync"} + 2023-08-15T01:41:01Z INFO mgmt loading http rules {"name": "alerting.node.NodeAlertingCapability", "rules": 1} + 2023-08-15T01:41:01Z DEBUG mgmt rule descriptors {"name": "alerting.node.NodeAlertingCapability", "rules": [{"http":{"Pattern":{"Post":"/v1/node/sync"},"body":"*"},"method":{"name":"Sync","input_type":".alerting.node.AlertingCapabilityConfig","output_type":".alerting.node.SyncResponse","options":{}}}]} + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/gateway.stream calling client handler + 2023-08-15T01:41:01Z INFO mgmt loading service {"name": "node.metrics.NodeConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading http rules {"name": "node.metrics.NodeConfiguration", "rules": 4} + 2023-08-15T01:41:01Z DEBUG mgmt rule descriptors {"name": "node.metrics.NodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".node.metrics.MetricsCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".node.metrics.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:41:01Z INFO plugin.metrics.cortex-http configuring http api server + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z INFO plugin.alerting using cluster driver {"driver": "test-environment"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /ready", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z INFO plugin.opni.alerting starting opni embedded server {"addr": "127.0.0.1:45261"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PUT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PATCH /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "HEAD /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "DELETE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "DELETE /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "TRACE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z INFO test.env Waiting for gateway to start... + 2023-08-15T01:41:01Z INFO test.env loaded 12 plugins + 2023-08-15T01:41:01Z INFO mgmt management HTTP server starting {"address": "localhost:42285"} + 2023-08-15T01:41:01Z INFO mgmt management gRPC server starting {"address": "127.0.0.1:43185"} + 2023-08-15T01:41:01Z INFO gateway.grpc gateway gRPC server starting {"address": "127.0.0.1:46393"} + 2023-08-15T01:41:01Z INFO gateway.http gateway HTTP server starting {"api": "127.0.0.1:45035", "metrics": "127.0.0.1:35949"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/tokens"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/tokens/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/watch"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/clusters/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/certs"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}/health"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/health/watch"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/clusters/{cluster.id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/roles"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/roles"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/roles/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/rolebindings"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/rolebindings"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/subjectaccess"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/apiextensions"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/config"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/config"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/capabilities"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/capabilities/{name}/installer"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/install"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/uninstall"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/status"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/status"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/cancel"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/dashboard/settings"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/dashboard/settings"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/all_user_stats"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/write_metrics"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules/{namespace}/{groupName}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/metadata"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules"} + 2023-08-15T01:41:01Z INFO test.env Gateway started + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/rules"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/CortexAdmin/rules/{groupName}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/flush_blocks"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/metadata"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/labels"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/status"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/config"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/raw"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertConditions/groups"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/configure"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertConditions/configure"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/choices"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/configure"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/status/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/withStatus"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/clone"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/silences"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/silences"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/timeline"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/list/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/toggleNotifications"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertEndpoints/list"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/delete"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/test"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/trigger"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/push"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/notifications/list"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/alarms/list"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/configuration"} + 2023-08-15T01:41:01Z INFO test starting agent {"id": "agent1", "address": "127.0.0.1:43839", "version": "v2"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/configure"} + 2023-08-15T01:41:01Z DEBUG agent using log level: debug + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/status"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/uninstall"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/configuration"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/configure"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/status"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/install"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/uninstall"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/info"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/RemoteReadGateway/targets/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/start/{target.meta.clusterId}/{target.meta.name}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/stop/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/status"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/edit"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config/{node.id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/NodeAlertingCapability/v1/node/sync"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config/{node.id}"} + 2023-08-15T01:41:01Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:41:01Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent1 + 2023-08-15T01:41:01Z INFO agent bootstrap completed successfully + 2023-08-15T01:41:01Z DEBUG keyring loaded ephemeral key {"path": "/tmp/opni-test-4032888549/keyring/local-agent.json", "usage": "auth", "labels": {"opni.io/session-attribute":"local"}} + 2023-08-15T01:41:01Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:01Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent1 is now active + 2023-08-15T01:41:01Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:41:01Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent1 is now active + 2023-08-15T01:41:01Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:41:01Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.apiextensions.StreamAPIExtension", "opni.apiextensions.HTTPAPIExtension", "opni.backends.Capability", "opni.Health"]} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:41:01Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:41:01Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:41:01Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:41:01Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:41:01Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:41:01Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:41:01Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:41:01Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:41:01Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:41:01Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:41:01Z INFO agent using unmanaged plugin loader + 2023-08-15T01:41:01Z INFO agent agent http server starting {"address": "127.0.0.1:43839"} + 2023-08-15T01:41:01Z INFO agent connecting to gateway... + 2023-08-15T01:41:01Z DEBUG gateway ratelimit: 50.000000 available + 2023-08-15T01:41:01Z INFO test starting agent {"id": "agent2", "address": "127.0.0.1:44837", "version": "v2"} + 2023-08-15T01:41:01Z DEBUG agent using log level: debug + 2023-08-15T01:41:01Z DEBUG gateway-client authenticated {"id": "agent1", "attributes": ["local"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream stream connected + 2023-08-15T01:41:01Z DEBUG gateway.grpc handling new stream connection + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent1"} + 2023-08-15T01:41:01Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent1"} + 2023-08-15T01:41:01Z DEBUG gateway.sync agent connected {"id": "agent1"} + 2023-08-15T01:41:01Z INFO gateway.monitor received status update {"id": "agent1", "connected": true, "attributes": ["local"]} + 2023-08-15T01:41:01Z DEBUG gateway.delegate agent connected {"id": "agent1"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:41:01Z INFO gateway.monitor received health update {"id": "agent1", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:41:01Z INFO agent gateway connected + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:41:01Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:41:01Z INFO agent bootstrap completed successfully + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent2 + 2023-08-15T01:41:01Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent1", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:41:01Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:41:01Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:01Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent2 is now active + 2023-08-15T01:41:01Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:41:01Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:41:01Z INFO gateway.monitor received health update {"id": "agent1", "ready": true, "conditions": []} + 2023-08-15T01:41:01Z DEBUG test.env configuring node {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:41:01Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:41:01Z DEBUG test.env configuring node {"node": "agent1", "driver": "otel"} + 2023-08-15T01:41:01Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "otel"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent1"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:41:01Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.HTTPAPIExtension"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:41:01Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent2 is now active + 2023-08-15T01:41:01Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting rule sync is disabled {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:41:01Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:41:01Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:41:01Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:41:01Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:41:01Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:41:01Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:41:01Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:41:01Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:41:01Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:41:01Z INFO agent using unmanaged plugin loader + 2023-08-15T01:41:01Z INFO agent agent http server starting {"address": "127.0.0.1:44837"} + 2023-08-15T01:41:01Z INFO agent connecting to gateway... + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:41:01Z DEBUG gateway ratelimit: 49.998515 available + 2023-08-15T01:41:01Z DEBUG gateway-client authenticated {"id": "agent2"} + 2023-08-15T01:41:01Z DEBUG gateway.grpc handling new stream connection + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent2"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream stream connected + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent2"} +• [0.346 seconds] +------------------------------ +Node Config when changing the default config should return the new config for all nodes [integration] +/home/ec2-user/opni/test/plugins/alerting/node_config_test.go:149 + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:41:01Z INFO gateway.monitor received status update {"id": "agent2", "connected": true} + 2023-08-15T01:41:01Z DEBUG gateway.sync agent connected {"id": "agent2"} + 2023-08-15T01:41:01Z DEBUG gateway.delegate agent connected {"id": "agent2"} + 2023-08-15T01:41:01Z INFO gateway.monitor received health update {"id": "agent2", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:41:01Z INFO agent gateway connected + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:41:01Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent2", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent2", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:41:01Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:41:01Z INFO gateway.monitor received health update {"id": "agent2", "ready": true, "conditions": []} + 2023-08-15T01:41:01Z DEBUG gateway.sync sending sync request to agent {"agentId": "", "capabilities": ["alerting"]} + 2023-08-15T01:41:01Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent2"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:41:01Z DEBUG test.env configuring node {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:41:01Z DEBUG test.env nothing to do (already stopped) {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:41:01Z DEBUG test.env configuring node {"node": "agent2", "driver": "otel"} + 2023-08-15T01:41:01Z DEBUG test.env nothing to do (already stopped) {"node": "agent2", "driver": "otel"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting rule sync is disabled {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:41:01Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:41:01Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:41:01Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG gateway.sync sending sync request to agent {"agentId": "", "capabilities": ["alerting"]} + 2023-08-15T01:41:01Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "(all)", "capability": "alerting"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:41:01Z INFO plugin.alerting alerting node config is up to date {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent2", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting node config is up to date {"component": "health-cfg-sync"} +• [0.063 seconds] +------------------------------ +Node Config when setting a config for a node should return the new config for that node [integration] +/home/ec2-user/opni/test/plugins/alerting/node_config_test.go:175 + 2023-08-15T01:41:01Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent1", "capabilities": ["alerting"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:41:01Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "agent1", "capability": "alerting"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting node config is up to date {"component": "health-cfg-sync"} +• [0.124 seconds] +------------------------------ +Node Config when resetting a config for a node should return the default config for that node [integration] +/home/ec2-user/opni/test/plugins/alerting/node_config_test.go:206 + 2023-08-15T01:41:01Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent1", "capabilities": ["alerting"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:41:01Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "agent1", "capability": "alerting"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting node config is up to date {"component": "health-cfg-sync"} +• [0.127 seconds] +------------------------------ +Node Config when resetting the default config should return the original default config for all nodes [integration] +/home/ec2-user/opni/test/plugins/alerting/node_config_test.go:231 + 2023-08-15T01:41:01Z DEBUG gateway.sync sending sync request to agent {"agentId": "", "capabilities": ["alerting"]} + 2023-08-15T01:41:01Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:41:01Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG gateway.sync sending sync request to agent {"agentId": "", "capabilities": ["alerting"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent2", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "(all)", "capability": "alerting"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting node config is up to date {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting node config is up to date {"component": "health-cfg-sync"} +• [0.030 seconds] +------------------------------ +Node Config when setting a config for a node that is the same as the default should preserve the config for that node if the default changes [integration] +/home/ec2-user/opni/test/plugins/alerting/node_config_test.go:250 + 2023-08-15T01:41:01Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent1", "capabilities": ["alerting"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:41:01Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "agent1", "capability": "alerting"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting node config is up to date {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:01Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:01Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:01Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:01Z DEBUG gateway.sync sending sync request to agent {"agentId": "", "capabilities": ["alerting"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:41:01Z DEBUG gateway.sync sending sync request to agent {"agentId": "", "capabilities": ["alerting"]} + 2023-08-15T01:41:01Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "(all)", "capability": "alerting"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting node config is up to date {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent2", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting node config is up to date {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting closing cortex cluster status watcher... {"watcher": "cortex-cluster-status"} + 2023-08-15T01:41:01Z INFO plugin.alerting shutting down cluster driver update handler + 2023-08-15T01:41:01Z INFO plugin.alerting exiting main sync loop + 2023-08-15T01:41:01Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:01Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error + 2023-08-15T01:41:01Z WARN plugin.alerting failed to receive cluster health status from grpc stream, retrying... + 2023-08-15T01:41:01Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:01Z INFO gateway shutting down plugins + 2023-08-15T01:41:01Z DEBUG gateway.sync agent disconnected {"id": "agent1"} + 2023-08-15T01:41:01Z DEBUG gateway.delegate agent disconnected {"id": "agent1"} + 2023-08-15T01:41:01Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:01Z INFO gateway.grpc agent stream closing {"error": "context canceled"} + 2023-08-15T01:41:01Z INFO gateway all plugins shut down + 2023-08-15T01:41:01Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:01Z ERROR plugin.alerting failed to receive cluster event : rpc error: code = Unavailable desc = error reading from server: read unix @->/tmp/plugin308284646: use of closed network connection + 2023-08-15T01:41:01Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error + 2023-08-15T01:41:01Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error + 2023-08-15T01:41:01Z ERROR plugin.alerting EOF + 2023-08-15T01:41:01Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error + 2023-08-15T01:41:01Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:01Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:41:01Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error + 2023-08-15T01:41:01Z DEBUG gateway.delegate agent disconnected {"id": "agent2"} + 2023-08-15T01:41:01Z DEBUG gateway.sync agent disconnected {"id": "agent2"} + 2023-08-15T01:41:01Z INFO gateway.grpc agent stream closing {"error": "context canceled"} + 2023-08-15T01:41:01Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:01Z WARN gateway http server exited with error {"error": "context canceled"} + 2023-08-15T01:41:01Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:01Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:41:01Z WARN test.env gateway server exited with error {"error": "context canceled"} + 2023-08-15T01:41:01Z ERROR test agent "agent2" exited: context canceled + 2023-08-15T01:41:01Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error + 2023-08-15T01:41:01Z ERROR test agent "agent1" exited: context canceled + 2023-08-15T01:41:01Z WARN test.env management server exited with error {"error": "context canceled"} +• [0.157 seconds] +------------------------------ +Alerting Cluster Integration tests when Installing the Alerting Cluster should install the alerting cluster [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:160 + 2023-08-15T01:41:01Z DEBUG plugin.alerting-test-cluster-driver Syncer start : alerting-server --syncer.alertmanager.config.file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml --syncer.listen.address=:43299 --syncer.alertmanager.address=http://127.0.0.1:39527 --syncer.gateway.join.address=:42179 syncer {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:01Z INFO plugin.alerting-test-cluster-driver Starting AlertManager {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml", "alertmanager-port": 39527, "opni-port": 43851} + ts=2023-08-15T01:41:01.848Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:41:01.848Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + ts=2023-08-15T01:41:01.851Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=43159 + ts=2023-08-15T01:41:01.852Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + ts=2023-08-15T01:41:01.885Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:01.885Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:01.888Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:39527 + ts=2023-08-15T01:41:01.888Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:39527 + 2023-08-15T01:41:02Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:02Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:02Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:02Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:02Z INFO plugin.alerting-test-cluster-driver Starting AlertManager Syncer {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml", "port": 43299} + 2023-08-15T01:41:02Z INFO plugin.alerting-test-cluster-driver AlertManager started {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml", "address": "http://127.0.0.1:39527"} + 2023-08-15T01:41:02Z INFO plugin.alerting updating known peers : [{http://127.0.0.1:39527 http://127.0.0.1:43869}] +• [1.020 seconds] +------------------------------ +Alerting Cluster Integration tests when Installing the Alerting Cluster should apply the configuration [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:176 + 2023-08-15T01:41:02Z DEBUG plugin.alerting-test-cluster-driver Syncer start : alerting-server --syncer.alertmanager.config.file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml --syncer.listen.address=:34301 --syncer.alertmanager.address=http://127.0.0.1:35817 --syncer.gateway.join.address=:42179 syncer {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:02Z INFO plugin.alerting-test-cluster-driver Starting AlertManager {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml", "alertmanager-port": 35817, "opni-port": 45277} + 2023-08-15T01:41:02Z DEBUG commands/alerting.go:72 syncer gateway join address:42179 + 2023-08-15T01:41:02Z INFO plugin.alerting-syncer syncer/syncer_server.go:90 starting alerting syncer server as identity 347a9035-d7e3-4fd9-82da-706a094b325b + ts=2023-08-15T01:41:02.863Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:41:02.863Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + ts=2023-08-15T01:41:02.866Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=35365 + 2023-08-15T01:41:02Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:108 acquired gateway alerting client + 2023-08-15T01:41:02Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:41:02Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:41:02Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:144 starting alerting syncer stream subscription + 2023-08-15T01:41:02Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "a2d53738-539c-49d0-9e03-5398bbd31505"} + 2023-08-15T01:41:02Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:150 starting new stream subscription + 2023-08-15T01:41:02Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:41:02Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + ts=2023-08-15T01:41:02.869Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + 2023-08-15T01:41:02Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "461299ca-49e7-4b9f-9be4-4294f13cf095"} + 2023-08-15T01:41:02Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:02Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:02.883Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:02.884Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:02Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:02.904Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:02.905Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:02.907Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:35817 + ts=2023-08-15T01:41:02.907Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:35817 + 2023-08-15T01:41:03Z INFO plugin.alerting-test-cluster-driver Starting AlertManager Syncer {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml", "port": 34301} + 2023-08-15T01:41:03Z INFO plugin.alerting-test-cluster-driver AlertManager started {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml", "address": "http://127.0.0.1:35817"} + 2023-08-15T01:41:03Z DEBUG plugin.alerting-test-cluster-driver Syncer start : alerting-server --syncer.alertmanager.config.file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml --syncer.listen.address=:46791 --syncer.alertmanager.address=http://127.0.0.1:36617 --syncer.gateway.join.address=:42179 syncer {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:03Z INFO plugin.alerting-test-cluster-driver Starting AlertManager {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml", "alertmanager-port": 36617, "opni-port": 41133} + ts=2023-08-15T01:41:03.853Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=2 elapsed=2.001124666s + 2023-08-15T01:41:03Z DEBUG commands/alerting.go:72 syncer gateway join address:42179 + 2023-08-15T01:41:03Z INFO plugin.alerting-syncer syncer/syncer_server.go:90 starting alerting syncer server as identity 718140ab-0c49-494d-907b-d6f92289873c + 2023-08-15T01:41:03Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:108 acquired gateway alerting client + 2023-08-15T01:41:03Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:41:03Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:41:03Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:144 starting alerting syncer stream subscription + 2023-08-15T01:41:03Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "1b66a932-21d1-4ce6-a23a-a823c793ec18"} + ts=2023-08-15T01:41:03.956Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:41:03.957Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + 2023-08-15T01:41:03Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:150 starting new stream subscription + 2023-08-15T01:41:03Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:41:03Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:41:03Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "eac5b487-db34-44da-8c98-ed6fea524cc5"} + ts=2023-08-15T01:41:03.965Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=41425 + ts=2023-08-15T01:41:03.968Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + 2023-08-15T01:41:03Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:03Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:03.972Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:03.972Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:03Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:04.004Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.005Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.007Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:36617 + ts=2023-08-15T01:41:04.007Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:36617 + 2023-08-15T01:41:04Z INFO plugin.alerting-test-cluster-driver Starting AlertManager Syncer {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml", "port": 46791} + 2023-08-15T01:41:04Z INFO plugin.alerting-test-cluster-driver AlertManager started {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml", "address": "http://127.0.0.1:36617"} + 2023-08-15T01:41:04Z INFO plugin.alerting updating known peers : [{http://127.0.0.1:39527 http://127.0.0.1:43869} {http://127.0.0.1:35817 http://127.0.0.1:43869} {http://127.0.0.1:36617 http://127.0.0.1:43869}] + 2023-08-15T01:41:04Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:04Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:04Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} +• [2.030 seconds] +------------------------------ +Alerting Cluster Integration tests when Installing the Alerting Cluster the alerting plugin components should be running and healthy [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:204 + 2023-08-15T01:41:04Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:04Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:04Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:04Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:04Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:04Z DEBUG api => {"method": "GET", "path": "/plugin_alerting/ready", "to": "127.0.0.1:35153 (plugin_alerting)", "for": "127.0.0.1", "host": "localhost:37611", "scheme": "https"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:04Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:04Z DEBUG api => {"method": "GET", "path": "/plugin_alerting/healthy", "to": "127.0.0.1:35153 (plugin_alerting)", "for": "127.0.0.1", "host": "localhost:37611", "scheme": "https"} +• [0.036 seconds] +------------------------------ +Alerting Cluster Integration tests when Installing the Alerting Cluster should be able to create some endpoints [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:239 + STEP: verifying they are externally persisted @ 08/15/23 01:41:04.85 + STEP: verifying they are reachable @ 08/15/23 01:41:04.857 + 2023-08-15T01:41:04Z DEBUG plugin.alerting sent manual sync request {"method": "sendManualSyncRequest", "sync-id": "de3890376c48dd11c94cefabc6a269ffaed3e7418bc3eef8bcc0b015d661aca1"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (de3890376c48dd11c94cefabc6a269ffaed3e7418bc3eef8bcc0b015d661aca1) config message + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received request to trigger alerts on condition id:"eCX4oAdVdZHpaC5g7h4xUX" {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received alert annotations : map[OpniHeader:Test notification OpniSummary:Admin has sent a test notification] {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (de3890376c48dd11c94cefabc6a269ffaed3e7418bc3eef8bcc0b015d661aca1) config message + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:04.870Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=3 elapsed=2.000885063s + ts=2023-08-15T01:41:04.870Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.871Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.871Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.871Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting sent manual sync request {"method": "sendManualSyncRequest", "sync-id": "f15b556b3ab0d0701569f65cbf4f08f871a4b9ddf73916e3f54c6d65b276a387"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received request to trigger alerts on condition id:"7kFfA7bgQ9ukcD3Zqo7h7Q" {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received alert annotations : map[OpniHeader:Test notification OpniSummary:Admin has sent a test notification] {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (f15b556b3ab0d0701569f65cbf4f08f871a4b9ddf73916e3f54c6d65b276a387) config message + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (f15b556b3ab0d0701569f65cbf4f08f871a4b9ddf73916e3f54c6d65b276a387) config message + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:04.884Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.884Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:04.885Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.885Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z DEBUG commands/alerting.go:72 syncer gateway join address:42179 + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:90 starting alerting syncer server as identity 4ea3286b-3229-421f-9efb-e5a702a331a8 + 2023-08-15T01:41:04Z DEBUG plugin.alerting sent manual sync request {"method": "sendManualSyncRequest", "sync-id": "2423c2d3586fb0857863f8e21bcf39c3c60b98341f8ba81189b64d3031b79c6d"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received request to trigger alerts on condition id:"SGuVCs9FnNm3UaHr6Bm6Y" {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received alert annotations : map[OpniHeader:Test notification OpniSummary:Admin has sent a test notification] {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (2423c2d3586fb0857863f8e21bcf39c3c60b98341f8ba81189b64d3031b79c6d) config message + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (2423c2d3586fb0857863f8e21bcf39c3c60b98341f8ba81189b64d3031b79c6d) config message + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:04.898Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.899Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.900Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:04.899Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:108 acquired gateway alerting client + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:144 starting alerting syncer stream subscription + 2023-08-15T01:41:04Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "91dd5c2f-3786-4562-870a-0f052b40ecbd"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:150 starting new stream subscription + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:41:04Z DEBUG plugin.alerting sent manual sync request {"method": "sendManualSyncRequest", "sync-id": "4775d46c83afd9b12874d7602b38590d5db61b43c982aab4018b64be7fbe4859"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received request to trigger alerts on condition id:"QMH3opSfy9EGrSBnyp56X4" {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received alert annotations : map[OpniHeader:Test notification OpniSummary:Admin has sent a test notification] {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "274ca105-9abb-4f72-a7e7-53dedf1980f4"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (4775d46c83afd9b12874d7602b38590d5db61b43c982aab4018b64be7fbe4859) config message + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (4775d46c83afd9b12874d7602b38590d5db61b43c982aab4018b64be7fbe4859) config message + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:04.909Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.909Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.909Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.909Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (127fe9181331fbe2847a95a2797367ef9dd375c862ac9b44cf9df5f5c023a3a6) config message + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:04.924Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.925Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting sent manual sync request {"method": "sendManualSyncRequest", "sync-id": "127fe9181331fbe2847a95a2797367ef9dd375c862ac9b44cf9df5f5c023a3a6"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received request to trigger alerts on condition id:"oc2ac5Wcs6uAzgERrTsdvc" {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received alert annotations : map[OpniHeader:Test notification OpniSummary:Admin has sent a test notification] {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (127fe9181331fbe2847a95a2797367ef9dd375c862ac9b44cf9df5f5c023a3a6) config message + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (127fe9181331fbe2847a95a2797367ef9dd375c862ac9b44cf9df5f5c023a3a6) config message + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:04.933Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.934Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.933Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.934Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (127fe9181331fbe2847a95a2797367ef9dd375c862ac9b44cf9df5f5c023a3a6) config message + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:05Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:05Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:05Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:05Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:05Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:05Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:05Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:05Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:05Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:05Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:05Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:05.631Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:05.631Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:05.631Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:05.631Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:05.632Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:05.632Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:05Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:05Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:05Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:05.854Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=1 before=2 now=3 elapsed=4.002109339s + 2023-08-15T01:41:05Z INFO plugin.alerting evaluation context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:41:05Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onCapabilityStatusCreate": "stdRzQww4KPXDy5r3dVAh6"} + ts=2023-08-15T01:41:05.969Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=3 elapsed=2.000338425s + 2023-08-15T01:41:06Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:06Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:06Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:06Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:06Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:06Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:06Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:06Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:06Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:06Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:06Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:07Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:07Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:07Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:07Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:07Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:07Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:07Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:07Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:07Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:07Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:07Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:08Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:08Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:08Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:08Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:08Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:08Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:08Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:08Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:08Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:08Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:08Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:09Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:09Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:09Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:09Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:09Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:09Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:09Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:09Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:09Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:09Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:09Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:10Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:10Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:10Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:10Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:10Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:10Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:10Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:10Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:10Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:10Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:10Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:10Z INFO plugin.alerting successfully synced (1/5) conditions {"component": "alarms"} + 2023-08-15T01:41:10Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 3/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:10Z ERROR plugin.alerting context canceled + nats: timeout {"action": "runSyncTasks"} + 2023-08-15T01:41:10Z ERROR plugin.alerting failed to successfully run all alerting sync tasks : context canceled + nats: timeout + 2023-08-15T01:41:10Z ERROR plugin.alerting skipping alerting periodic sync due to error : %scontext canceled {"action": "runSyncTasks"} + 2023-08-15T01:41:10Z ERROR plugin.alerting failed to successfully run all alerting sync tasks : context canceled + 2023-08-15T01:41:10Z INFO plugin.alerting exiting main sync loop + 2023-08-15T01:41:11Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:11Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:11Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:11Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:11Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:11Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:11Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:11Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:11Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:11Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:11Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:12Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:12Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:12Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:12Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:12Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:12Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:12Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:12Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:12Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:12Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:12Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + ts=2023-08-15T01:41:12.873Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.004204209s + 2023-08-15T01:41:13Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:13Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:13Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:13Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:13Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:13Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:13Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:13Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:13Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:13Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:13Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + ts=2023-08-15T01:41:13.856Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=12.00388903s + ts=2023-08-15T01:41:13.857Z caller=notify.go:740 level=warn component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 127.0.0.1:3000: connect: connection refused" + ts=2023-08-15T01:41:13.971Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.00257536s + 2023-08-15T01:41:14Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:14Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:14Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:14Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:14Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:14Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:14Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:14Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:14Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:14Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:14Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:15Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:15Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:15Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:15Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:15Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:15Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:15Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:15Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:15Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:15Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:15Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:16Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:16Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:16Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:16Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:16Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:16Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:16Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:16Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:16Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:16Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:16Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:17Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:17Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:17Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:17Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:17Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:17Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:17Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:17Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:17Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:17Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:17Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:18Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:18Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:18Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:18Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:18Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:18Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:18Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:18Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:18Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:18Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:18Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:19Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:19Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:19Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:19Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:19Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:19Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:19Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:19Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:19Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:19Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:19Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:20Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:20Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:20Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:20Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:20Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:20Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:20Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:20Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:20Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:20Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:20Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:21Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:21Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:21Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:21Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:21Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:21Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:21Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:21Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:21Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:21Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:21Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:22Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:22Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:22Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:22Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:22Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:22Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:22Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:22Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:22Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:22Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:22Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:23Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:23Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:23Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:23Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:23Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:23Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:23Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:23Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:23Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:23Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:23Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:24Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:24Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:24Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:24Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:24Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:24Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:24Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:24Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:24Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:24Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:24Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:25Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:25Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:25Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:25Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:25Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:25Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:25Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:25Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:25Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:25Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:25Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:26Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:26Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:26Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:26Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:26Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:26Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:26Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:26Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:26Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:26Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:26Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:27Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:27Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:27Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:27Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:27Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:27Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:27Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:27Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:27Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:27Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:27Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:28Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:28Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:28Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:28Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:28Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:28Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:28Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:28Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:28Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:28Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:28Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:29Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:29Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:29Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:29Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:29Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:29Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:29Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:29Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:29Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:29Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:29Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:30Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:30Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:30Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:30Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:30Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:30Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:30Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:30Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:30Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:30Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:30Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:31Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:31Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:31Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:31Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:31Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:31Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:31Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:31Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:31Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:31Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:31Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:32Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:32Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:32Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:32Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:32Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:32Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:32Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:32Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:32Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:32Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:32Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:33Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:33Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:33Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:33Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:33Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:33Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:33Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:33Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:33Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:33Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:33Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:34Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:34Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:34Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:34Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:34Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:34Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:34Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:34Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:34Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:34Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:34Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + [FAILED] in [It] - /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:269 @ 08/15/23 01:41:34.934 + [FAILED] in [AfterEach] - /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:153 @ 08/15/23 01:41:34.944 +• [FAILED] [30.109 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should be able to create some endpoints [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:239 + + [FAILED] Timed out after 30.001s. + Expected success, but got an error: + <*errors.errorString | 0xc0054da220>: + server mock-integration-fAeNNrkhJqtd26oSstsDta did not receive any alerts + { + s: "server mock-integration-fAeNNrkhJqtd26oSstsDta did not receive any alerts", + } + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:269 @ 08/15/23 01:41:34.934 + + There were additional failures detected. To view them in detail run ginkgo -vv +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should create some default conditions when bootstrapping agents [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:277 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:277 @ 08/15/23 01:41:34.945 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] shoud list conditions by given filters [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:335 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:335 @ 08/15/23 01:41:34.946 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should be able to attach endpoints to conditions [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:401 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:401 @ 08/15/23 01:41:34.947 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] agent disconnect alarms should fire when agents are disconnected  [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:505 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:505 @ 08/15/23 01:41:34.948 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should be able to batch list status and filter by status [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:601 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:601 @ 08/15/23 01:41:34.948 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should be able to push notifications to our notification endpoints [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:627 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:627 @ 08/15/23 01:41:34.949 +------------------------------ +P [PENDING] +Alerting Cluster Integration tests when Installing the Alerting Cluster should be able to list opni messages [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:651 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should return warnings when trying to edit/delete alert endpoints that are involved in conditions [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:671 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:671 @ 08/15/23 01:41:34.95 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should have a functional timeline [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:706 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:706 @ 08/15/23 01:41:34.951 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] the alertmanager proxy served by the Gateway HTTP port should be able to list the alarms [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:757 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:757 @ 08/15/23 01:41:34.952 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should sync friendly cluster names to conditions [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:782 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:782 @ 08/15/23 01:41:34.953 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should force update/delete alert endpoints involved in conditions [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:818 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:818 @ 08/15/23 01:41:34.953 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should delete the downstream agents [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:881 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:881 @ 08/15/23 01:41:34.954 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should uninstall the alerting cluster [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:891 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:891 @ 08/15/23 01:41:34.955 +------------------------------ +[DeferCleanup (Suite)]  +/home/ec2-user/opni/test/plugins/alerting/alerting_suite_test.go:51 + 2023-08-15T01:41:34Z DEBUG plugin.alerting closing cortex cluster status watcher... {"watcher": "cortex-cluster-status"} + 2023-08-15T01:41:34Z INFO plugin.alerting exiting main sync loop + 2023-08-15T01:41:34Z INFO plugin.alerting shutting down cluster driver update handler + 2023-08-15T01:41:34Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "461299ca-49e7-4b9f-9be4-4294f13cf095"} + 2023-08-15T01:41:34Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:34Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "a2d53738-539c-49d0-9e03-5398bbd31505"} + 2023-08-15T01:41:34Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "91dd5c2f-3786-4562-870a-0f052b40ecbd"} + 2023-08-15T01:41:34Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "1b66a932-21d1-4ce6-a23a-a823c793ec18"} + 2023-08-15T01:41:34Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "eac5b487-db34-44da-8c98-ed6fea524cc5"} + 2023-08-15T01:41:34Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "274ca105-9abb-4f72-a7e7-53dedf1980f4"} + 2023-08-15T01:41:34Z WARN plugin.alerting failed to receive cluster health status from grpc stream, retrying... + ts=2023-08-15T01:41:34.957Z caller=alertmanager_main.go:601 level=info msg="Received SIGTERM, exiting gracefully..." + 2023-08-15T01:41:34Z ERROR plugin.alerting failed to receive cluster event : rpc error: code = Canceled desc = context canceled + ts=2023-08-15T01:41:34.957Z caller=alertmanager_main.go:601 level=info msg="Received SIGTERM, exiting gracefully..." + 2023-08-15T01:41:34Z INFO gateway shutting down plugins + 2023-08-15T01:41:34Z WARN test.env management server exited with error {"error": "context canceled"} + 2023-08-15T01:41:34Z WARN gateway http server exited with error {"error": "context canceled"} + 2023-08-15T01:41:34Z WARN test.env gateway server exited with error {"error": "context canceled"} + 2023-08-15T01:41:34Z INFO gateway all plugins shut down + 2023-08-15T01:41:34Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:34Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:34Z ERROR plugin.alerting EOF +[DeferCleanup (Suite)] PASSED [0.024 seconds] +------------------------------ +[DeferCleanup (Suite)]  +/home/ec2-user/go/pkg/mod/github.com/golang/mock@v1.7.0-rc.1/gomock/controller.go:99 +[DeferCleanup (Suite)] PASSED [0.000 seconds] +------------------------------ + +Summarizing 1 Failure: + [FAIL] Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should be able to create some endpoints [integration] + /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:269 + +Ran 27 of 41 Specs in 99.430 seconds +FAIL! -- 26 Passed | 1 Failed | 1 Pending | 13 Skipped +--- FAIL: TestAlerting (99.43s) +FAIL + +Ginkgo ran 1 suite in 1m49.83826659s + +Test Suite Failed diff --git a/test/plugins/metrics/slo_test.go b/test/plugins/metrics/slo_test.go new file mode 100644 index 0000000000..1846d0aeba --- /dev/null +++ b/test/plugins/metrics/slo_test.go @@ -0,0 +1,567 @@ +package metrics_test + +import ( + "context" + "fmt" + "net/url" + "time" + + "github.com/golang/snappy" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + prommodel "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/rulefmt" + "github.com/prometheus/prometheus/prompb" + "github.com/prometheus/prometheus/storage/remote" + capabilityv1 "github.com/rancher/opni/pkg/apis/capability/v1" + corev1 "github.com/rancher/opni/pkg/apis/core/v1" + managementv1 "github.com/rancher/opni/pkg/apis/management/v1" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + storagev1 "github.com/rancher/opni/pkg/apis/storage/v1" + "github.com/rancher/opni/pkg/capabilities" + "github.com/rancher/opni/pkg/capabilities/wellknown" + "github.com/rancher/opni/pkg/logger" + "github.com/rancher/opni/pkg/metrics/compat" + "github.com/rancher/opni/pkg/slo/backend" + "github.com/rancher/opni/pkg/slo/backend/metrics" + "github.com/rancher/opni/pkg/test" + "github.com/rancher/opni/pkg/util" + "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" + "github.com/rancher/opni/plugins/metrics/apis/cortexops" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/timestamppb" + "gopkg.in/yaml.v3" +) + +func init() { + metrics.DefaultEvaluationInterval = 1 * time.Second +} + +var _ = Describe("Prometheus SLOs", Ordered, Label("integration"), func() { + var env *test.Environment + var fingerprint string + var adminClient cortexadmin.CortexAdminClient + var sloGen *metrics.SLOGeneratorImpl + var sloDatasource backend.SLODatasource + var rw remote.WriteClient + BeforeAll(func() { + env = &test.Environment{} + Expect(env.Start()).To(Succeed()) + DeferCleanup(env.Stop) + mgmtClient := env.NewManagementClient() + var err error + sloGen, err = metrics.NewSLOGenerator(*backend.SLODataToStruct(&slov1.SLOData{ + Id: "id1", + SLO: util.ProtoClone(tempSLO), + })) + Expect(err).To(BeNil()) + + certsInfo, err := mgmtClient.CertsInfo(context.Background(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + fingerprint = certsInfo.Chain[len(certsInfo.Chain)-1].Fingerprint + Expect(fingerprint).NotTo(BeEmpty()) + + token, err := mgmtClient.CreateBootstrapToken(context.Background(), &managementv1.CreateBootstrapTokenRequest{ + Ttl: durationpb.New(1 * time.Hour), + }) + Expect(err).NotTo(HaveOccurred()) + + agentPort, errC := env.StartAgent("agent", token, []string{fingerprint}) + Eventually(errC).Should(Receive(BeNil())) + + rw, err = remote.NewWriteClient("agent", &remote.ClientConfig{ + URL: &config.URL{URL: util.Must(url.Parse(fmt.Sprintf("http://127.0.0.1:%d/api/agent/push", agentPort)))}, + Timeout: model.Duration(time.Second * 10), + }) + Expect(err).NotTo(HaveOccurred()) + + opsClient := cortexops.NewCortexOpsClient(env.ManagementClientConn()) + _, err = opsClient.ConfigureCluster(context.Background(), &cortexops.ClusterConfiguration{ + Mode: cortexops.DeploymentMode_AllInOne, + Storage: &storagev1.StorageSpec{ + Backend: storagev1.Filesystem, + }, + }) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() error { + clusters, err := mgmtClient.ListClusters(env.Context(), &managementv1.ListClustersRequest{}) + if err != nil { + return err + } + if len(clusters.GetItems()) == 0 { + return fmt.Errorf("no clusters found") + } + return nil + }).Should(Succeed()) + + resp, err := mgmtClient.InstallCapability(context.Background(), &managementv1.CapabilityInstallRequest{ + Name: "metrics", + Target: &capabilityv1.InstallRequest{ + Cluster: &corev1.Reference{ + Id: "agent", + }, + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(resp.Status).To(Equal(capabilityv1.InstallResponseStatus_Success)) + + lg := logger.NewPluginLogger().Named("metrics") + + adminClient = cortexadmin.NewCortexAdminClient(env.ManagementClientConn()) + metricsProvider := metrics.NewProvider(lg.With("component", "slo")) + metricsProvider.Initialize(adminClient) + metricsStoreProvider := metrics.NewMetricsSLOStoreProvider(lg.With("component", "slo")) + metricsStoreProvider.Initialize(adminClient) + + sloDatasource = backend.NewSLODatasource( + metrics.NewMetricsSLOStore( + metricsStoreProvider, + ), + metrics.NewBackend( + metricsProvider, + ), + func(ctx context.Context, clusterId *corev1.Reference) error { + cluster, err := mgmtClient.GetCluster(ctx, &corev1.Reference{ + Id: clusterId.Id, + }) + if err != nil { + return err + } + if !capabilities.Has(cluster, capabilities.Cluster(wellknown.CapabilityMetrics)) { + return fmt.Errorf("requires metrics capability to be installed on cluster %s", cluster.GetId()) + } + return nil + }, + ) + }) + When("we use the prometheus slo generator", func() { + Specify("cortex should be reachable", func() { + Eventually(func() error { + statsList, err := adminClient.AllUserStats(context.Background(), &emptypb.Empty{}) + if err != nil { + return err + } + if len(statsList.Items) == 0 { + return fmt.Errorf("no user stats") + } + if statsList.Items[0].NumSeries == 0 { + return fmt.Errorf("no series") + } + return nil + }, 60*time.Second, 100*time.Millisecond).Should(Succeed()) + }) + It("should construct loadable SLI info rule", func() { + expectLoadRules(env.Context(), "info", adminClient, sloGen.Info()) + }) + + It("should construct loadable SLI period rule ", func() { + expectLoadRules(env.Context(), "period", adminClient, sloGen.Period()) + }) + + It("should construct loadable SLI error budget rule", func() { + expectLoadRules(env.Context(), "errbudget", adminClient, sloGen.ErrorBudget()) + }) + + It("should construct loadable SLI objective rule", func() { + expectLoadRules(env.Context(), "objective", adminClient, sloGen.Objective()) + }) + + It("should construct loadable SLI SLI rule", func() { + expectLoadRules(env.Context(), "objective", adminClient, sloGen.SLI(prommodel.Duration(time.Second*5))) + }) + + It("should construct SLI current burn rate rule", func() { + expectLoadRules(env.Context(), "currentBurnRate", adminClient, sloGen.CurrentBurnRate()) + }) + + It("should construct SLI period burn rate rule", func() { + expectLoadRules(env.Context(), "periodBurnRate", adminClient, sloGen.PeriodBurnRate()) + }) + + It("should construct error budget remaining rule", func() { + expectLoadRules(env.Context(), "errorBudgetRemaining", adminClient, sloGen.ErrorBudgetRemaining()) + }) + }) + + When("we use the SLO monitoring datasource service discovery backend", func() { + It("should remote write SLO metric data to cortex", func() { + now := time.Now().UnixMilli() + err := remoteWrite(env.Context(), rw, &prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "testmetric"}, + {Name: "job", Value: "testservice"}, + {Name: "a", Value: "a"}, + }, + Samples: []prompb.Sample{{Value: 1, Timestamp: now}}, + }, + }, + }) + Expect(err).To(Succeed()) + }) + + It("should list available services", func() { + Eventually(func() error { + svcList, err := sloDatasource.ListServices(env.Context(), &slov1.ListServicesRequest{ + Datasource: "monitoring", + ClusterId: "agent", + }) + if err != nil { + return err + } + if len(svcList.Items) == 0 { + return fmt.Errorf("no services") + } + if !svcList.ContainsId("prometheus") { + return fmt.Errorf("missing prometheus") + } + if !svcList.ContainsId("testservice") { + return fmt.Errorf("missing testservice") + } + return nil + }).Should(Succeed()) + }) + + It("should list available metrics on a service", func() { + Eventually(func() error { + metricList, err := sloDatasource.ListMetrics(env.Context(), &slov1.ListMetricsRequest{ + Datasource: "monitoring", + ClusterId: "agent", + ServiceId: "testservice", + }) + if err != nil { + return err + } + + if len(metricList.GroupNameToMetrics) == 0 { + return fmt.Errorf("no metrics") + } + + if !metricList.ContainsId("testmetric") { + return fmt.Errorf("missing testmetric") + } + return nil + }).Should(Succeed()) + + }) + + It("should list available events on a (metrics,service) pair", func() { + Eventually(func() error { + eventList, err := sloDatasource.ListEvents(env.Context(), &slov1.ListEventsRequest{ + Datasource: "monitoring", + ClusterId: "agent", + ServiceId: "testservice", + MetricId: "testmetric", + }) + if err != nil { + return err + } + if len(eventList.Items) == 0 { + return fmt.Errorf("no events") + } + if !eventList.ContainsId("a") { + return fmt.Errorf("missing event a") + } + return nil + }).Should(Succeed()) + + }) + }) + + When("we create SLO data", func() { + It("should write SLO data", func() { + dur := prommodel.Duration(time.Hour * 2) + err := writeSLO( + env.Context(), + rw, + time.Duration(dur), + "http_slo_response", + "service", + "code", + "200", + "500", + 99.9, + 10000, + ) + Expect(err) + }) + + It("should discover the previously written SLO data", func() { + Eventually(func() error { + svcList, err := sloDatasource.ListServices(env.Context(), &slov1.ListServicesRequest{ + Datasource: "monitoring", + ClusterId: "agent", + }) + if err != nil { + return err + } + if len(svcList.Items) == 0 { + return fmt.Errorf("no services") + } + + if !svcList.ContainsId("service") { + return fmt.Errorf("missing service") + } + return nil + }).Should(Succeed()) + + Eventually(func() error { + metricList, err := sloDatasource.ListMetrics(env.Context(), &slov1.ListMetricsRequest{ + Datasource: "monitoring", + ClusterId: "agent", + ServiceId: "service", + }) + if err != nil { + return err + } + + if len(metricList.GroupNameToMetrics) == 0 { + return fmt.Errorf("no metrics") + } + + if !metricList.ContainsId("http_slo_response") { + return fmt.Errorf("missing http_slo_response") + } + return nil + }).Should(Succeed()) + + Eventually(func() error { + eventList, err := sloDatasource.ListEvents(env.Context(), &slov1.ListEventsRequest{ + Datasource: "monitoring", + ClusterId: "agent", + ServiceId: "service", + MetricId: "http_slo_response", + }) + if err != nil { + return err + } + if len(eventList.Items) == 0 { + return fmt.Errorf("no events") + } + if !eventList.ContainsId("code") { + return fmt.Errorf("missing event 'code'") + } + return nil + }).Should(Succeed()) + }) + + It("should be able to query the written data", func() { + resp, err := adminClient.QueryRange(env.Context(), &cortexadmin.QueryRangeRequest{ + Tenants: []string{ + "agent", + }, + Query: "http_slo_response{code=\"200\"}", + Start: timestamppb.New(time.Now().Add(-time.Hour)), + End: timestamppb.New(time.Now()), + Step: durationpb.New(time.Second * 1), + }) + Expect(err).To(Succeed()) + Expect(resp).NotTo(BeNil()) + GinkgoWriter.Write(resp.Data) + qr, err := compat.UnmarshalPrometheusResponse(resp.Data) + Expect(err).To(Succeed()) + samples := qr.MapToSamples() + Expect(samples).NotTo(HaveLen(0)) + + resp2, err := adminClient.QueryRange(env.Context(), &cortexadmin.QueryRangeRequest{ + Tenants: []string{ + "agent", + }, + Query: "http_slo_response{code=\"500\"}", + Start: timestamppb.New(time.Now().Add(-time.Hour)), + End: timestamppb.New(time.Now()), + Step: durationpb.New(time.Second * 1), + }) + Expect(err).To(Succeed()) + Expect(resp2).NotTo(BeNil()) + GinkgoWriter.Write(resp2.Data) + qr2, err := compat.UnmarshalPrometheusResponse(resp2.Data) + Expect(err).To(Succeed()) + samples2 := qr2.MapToSamples() + Expect(samples2).NotTo(HaveLen(0)) + }) + + XIt("should create an SLO based on the written data", func() { + req := &slov1.CreateSLORequest{ + Slo: util.ProtoClone(metricsSlo), + } + plotVec, err := sloDatasource.Preview(env.Context(), req) + Expect(err).To(Succeed()) + Expect(plotVec).NotTo(BeNil()) + Expect(plotVec.PlotVector.Items).NotTo(HaveLen(0)) + + for _, item := range plotVec.PlotVector.Items { + Expect(item).NotTo(BeNil()) + Expect(item.Sli).To(BeNumerically(">", float64(0))) //FIXME: it is 0 right now + Expect(item.Sli).To(BeNumerically("<", float64(100))) + } + + ref, err := sloDatasource.Create(env.Context(), req) + Expect(err).To(Succeed()) + Expect(ref).NotTo(BeNil()) + Expect(ref.Id).NotTo(BeEmpty()) + }) + }) +}) + +func expectLoadRules( + ctx context.Context, + ruleGroupName string, + adminClient cortexadmin.CortexAdminClient, + gen metrics.MetricGenerator, +) { + periodRule, err := gen.Rule() + Expect(err).To(Succeed()) + + periodGroup := rulefmt.RuleGroup{ + Name: ruleGroupName, + Interval: prommodel.Duration(1 * time.Second), + Rules: []rulefmt.RuleNode{periodRule}, + } + + periodData, err := yaml.Marshal(periodGroup) + Expect(err).To(Succeed()) + + _, err = adminClient.LoadRules(ctx, &cortexadmin.LoadRuleRequest{ + ClusterId: "agent", + Namespace: "test-slo", + YamlContent: periodData, + }) + Expect(err).To(Succeed()) +} + +func remoteWrite(ctx context.Context, rw remote.WriteClient, req *prompb.WriteRequest) error { + wrData, err := req.Marshal() + Expect(err).NotTo(HaveOccurred()) + compressed := snappy.Encode(nil, wrData) + + return rw.Store(ctx, compressed) +} + +func writeSLO( + ctx context.Context, + rw remote.WriteClient, + period time.Duration, + metricName, serviceName, eventName, goodEvent, totalEvent string, + expectedSLI float64, + numSamples int, +) error { + if expectedSLI > 100 && expectedSLI < 0 { + return fmt.Errorf("expected SLI must be between 0 and 100") + } + + now := time.Now() + start := now.Add(-period) + + goodTs := []int64{} + goodSamples := int(float64(numSamples) * (expectedSLI / 100)) + goodStepDuration := int(period.Milliseconds()) / goodSamples + totalSamples := int(float64(numSamples) * ((100 - expectedSLI) / 100)) + totalStepDuration := int(period.Milliseconds()) / totalSamples + for i := 0; i < goodSamples; i++ { + toAdd := time.Duration(i*goodStepDuration) * time.Millisecond + goodTs = append(goodTs, start.Add(toAdd).UnixMilli()) + } + totalTs := []int64{} + for i := 0; i < totalSamples; i++ { + + toAdd := time.Duration(i*totalStepDuration) * time.Millisecond + totalTs = append(totalTs, start.Add(toAdd).UnixMilli()) + } + + if len(goodTs) == 0 { + panic("bug : test should have non-zero good event samples") + } + + if len(totalTs) == 0 { + panic("bug : test should have non-zero total event samples") + } + + wreq := &prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{}, + } + + goodEvents := prompb.TimeSeries{ + Labels: []prompb.Label{ + {Name: "__name__", Value: metricName}, + {Name: "job", Value: serviceName}, + {Name: eventName, Value: goodEvent}, + }, + Samples: []prompb.Sample{}, + } + totalEvents := prompb.TimeSeries{ + Labels: []prompb.Label{ + {Name: "__name__", Value: metricName}, + {Name: "job", Value: serviceName}, + {Name: eventName, Value: totalEvent}, + }, + } + + for i, ts := range goodTs { + goodEvents.Samples = append(goodEvents.Samples, prompb.Sample{ + Value: float64(i + 1), + Timestamp: ts, + }) + } + + for i, ts := range totalTs { + totalEvents.Samples = append(totalEvents.Samples, prompb.Sample{ + Value: float64(i + 1), + Timestamp: ts, + }) + } + + wreq.Timeseries = append(wreq.Timeseries, goodEvents) + wreq.Timeseries = append(wreq.Timeseries, totalEvents) + return remoteWrite(ctx, rw, wreq) +} + +var ( + tempSLO = &slov1.ServiceLevelObjective{ + Name: "test-slo-success", + ClusterId: "agent", + ServiceId: "testservice", + GoodMetricName: "testmetric", + GoodEvents: []*slov1.Event{ + { + Key: "code", + Vals: []string{"200"}, + }, + }, + TotalMetricName: "testmetric", + TotalEvents: []*slov1.Event{}, + Target: &slov1.Target{Value: 99.9}, + SloPeriod: "30d", + BudgetingInterval: durationpb.New(time.Minute * 5), + Datasource: "monitoring", + } + + metricsSlo = &slov1.ServiceLevelObjective{ + Name: "some-opaque-name", + Datasource: "monitoring", + ClusterId: "agent", + ServiceId: "service", + GoodMetricName: "http_slo_response", + GoodEvents: []*slov1.Event{ + { + Key: "code", + Vals: []string{"200"}, + }, + }, + TotalMetricName: "http_slo_response", + TotalEvents: []*slov1.Event{ + { + Key: "code", + Vals: []string{"500"}, + }, + }, + SloPeriod: "2h", + Target: &slov1.Target{Value: 99.9}, + BudgetingInterval: durationpb.New(time.Second * 1), + } +) diff --git a/test/plugins/slo/slo_suite_test.go b/test/plugins/slo/slo_suite_test.go deleted file mode 100644 index 38db6dd25e..0000000000 --- a/test/plugins/slo/slo_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package plugins_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - _ "github.com/rancher/opni/pkg/test/setup" - _ "github.com/rancher/opni/plugins/alerting/test" - _ "github.com/rancher/opni/plugins/metrics/test" - _ "github.com/rancher/opni/plugins/slo/test" -) - -func TestSloPlugin(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Plugins Suite") -} diff --git a/test/plugins/slo/slo_test.go b/test/plugins/slo/slo_test.go deleted file mode 100644 index 71b4d5f1c6..0000000000 --- a/test/plugins/slo/slo_test.go +++ /dev/null @@ -1,688 +0,0 @@ -package plugins_test - -import ( - "context" - "fmt" - "math" - "net/http" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - v1 "github.com/rancher/opni/pkg/apis/capability/v1" - corev1 "github.com/rancher/opni/pkg/apis/core/v1" - managementv1 "github.com/rancher/opni/pkg/apis/management/v1" - storagev1 "github.com/rancher/opni/pkg/apis/storage/v1" - "github.com/rancher/opni/pkg/capabilities/wellknown" - "github.com/rancher/opni/pkg/slo/query" - "github.com/rancher/opni/pkg/slo/shared" - "github.com/rancher/opni/pkg/test" - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - "github.com/rancher/opni/plugins/metrics/apis/cortexops" - sloapi "github.com/rancher/opni/plugins/slo/apis/slo" - "google.golang.org/protobuf/types/known/durationpb" - "google.golang.org/protobuf/types/known/emptypb" -) - -func canReachInstrumentationMetrics(instrumentationServerPort int) bool { - resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/metrics", instrumentationServerPort)) - if err != nil { - panic(err) - } - return resp.StatusCode == 200 -} - -func simulateGoodEvents(metricName string, instrumentationServerPort int, numEvents int) { - for i := 0; i < numEvents; i++ { - client := &http.Client{ - Transport: &http.Transport{}, - } - req, _ := http.NewRequest("GET", fmt.Sprintf("http://127.0.0.1:%d/%s/good", instrumentationServerPort, metricName), nil) - req.Close = true - resp, err := client.Do(req) - if err != nil { - panic(err) - } - resp.Body.Close() - if resp.StatusCode != 200 { - panic(resp.StatusCode) - } - } -} - -func simulateBadEvents(metricName string, instrumentationServerPort int, numEvents int) { - - for i := 0; i < numEvents; i++ { - client := &http.Client{ - Transport: &http.Transport{}, - } - req, _ := http.NewRequest("GET", fmt.Sprintf("http://127.0.0.1:%d/%s/bad", instrumentationServerPort, metricName), nil) - resp, err := client.Do(req) - if err != nil { - panic(err) - } - resp.Body.Close() - if resp.StatusCode != 200 { - panic(resp.StatusCode) - } - } -} - -// populate instrumentation server with good events -func simulateGoodStatus(metricName string, instrumentationServerPort int, numEvents int) (goodEventsCount int) { - simulateGoodEvents(metricName, instrumentationServerPort, numEvents) - return numEvents -} - -// populate instrumentation server with enough bad events to trigger an alerts, -// but maintain the current objective so we don't trigger a breaching status -// -// @warning : assumes slo objective is 0 <= x <= 100 -func simulateAlertingStatus( - metricName string, - instrumentationServerPort int, - numExistingGoodEvents int, - sloObjective float64) (currentEventRatio float64, numTotalEvents int) { - - // solve an equation for the good events / total events ratio >= sloObjective - // but gets as close as possible to the sloObjective - - if numExistingGoodEvents >= 0 { - panic("Need existing number of good events to easily calculate an alerting status") - } - - totalEventCount := int((sloObjective * 100) * 100) - remainingToAssign := totalEventCount - numExistingGoodEvents - // round up to ensure never triggering an SLO breach - goodEventsNum := int(math.Ceil(float64(remainingToAssign) * (sloObjective / 100))) - badEventsNum := remainingToAssign - goodEventsNum - simulateGoodEvents(metricName, instrumentationServerPort, goodEventsNum) - simulateBadEvents(metricName, instrumentationServerPort, badEventsNum) - // return base 100 ratio of good events to total events - return float64( - (numExistingGoodEvents+goodEventsNum)/(numExistingGoodEvents+goodEventsNum+badEventsNum)) * 100, - numExistingGoodEvents + goodEventsNum + badEventsNum -} - -/* -Simulate a breaching status, by adding enough bad events to fail the objective -*/ -func simulateBreachingStatus(metricName string, instrumentationServerPort int, - sloObjective float64, curEventRatio float64, totalEvents int) { - if curEventRatio >= sloObjective { - panic("Expected to at least be in a good/alerting status to simulate a breaching status") - } - ratioToBreach := sloObjective - curEventRatio - numEventsToBreach := int(math.Ceil(float64(totalEvents)*(ratioToBreach/100))) + 500 /* for good measure :) */ - simulateBadEvents(metricName, instrumentationServerPort, numEventsToBreach) -} - -var _ = XDescribe("Converting ServiceLevelObjective Messages to Prometheus Rules", Ordered, Label("integration", "slow"), func() { - ctx := context.Background() - // test environment references - var env *test.Environment - var sloClient sloapi.SLOClient - var adminClient cortexadmin.CortexAdminClient - var client managementv1.ManagementClient - // downstream server ports - var instrumentationPort int - var done chan struct{} - var token *corev1.BootstrapToken - var info *managementv1.CertsInfoResponse - - BeforeAll(func() { - env = &test.Environment{} - Expect(env.Start()).To(Succeed()) - DeferCleanup(env.Stop) - - opsClient := cortexops.NewCortexOpsClient(env.ManagementClientConn()) - _, err := opsClient.ConfigureCluster(context.Background(), &cortexops.ClusterConfiguration{ - Mode: cortexops.DeploymentMode_AllInOne, - Storage: &storagev1.StorageSpec{ - Backend: storagev1.Filesystem, - }, - }) - Expect(err).NotTo(HaveOccurred()) - - client = env.NewManagementClient() - token, err = client.CreateBootstrapToken(context.Background(), &managementv1.CreateBootstrapTokenRequest{ - Ttl: durationpb.New(time.Hour), - }) - Expect(err).NotTo(HaveOccurred()) - info, err = client.CertsInfo(context.Background(), &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - instrumentationPort, done = env.StartInstrumentationServer() - DeferCleanup(func() { - done <- struct{}{} - }) - _, errC := env.StartAgent("agent", token, []string{info.Chain[len(info.Chain)-1].Fingerprint}) - Eventually(errC).Should(Receive(BeNil())) - env.SetPrometheusNodeConfigOverride("agent", test.NewOverridePrometheusConfig( - "slo/prometheus/config.yaml", - []test.PrometheusJob{ - { - JobName: query.MockTestServerName, - ScrapePort: instrumentationPort, - }, - }), - ) - - _, errC = env.StartAgent("agent2", token, []string{info.Chain[len(info.Chain)-1].Fingerprint}) - Eventually(errC).Should(Receive(BeNil())) - - env.SetPrometheusNodeConfigOverride("agent2", test.NewOverridePrometheusConfig( - "slo/prometheus/config.yaml", - []test.PrometheusJob{ - { - JobName: query.MockTestServerName, - ScrapePort: instrumentationPort, - }, - }), - ) - - client.InstallCapability(context.Background(), &managementv1.CapabilityInstallRequest{ - Name: wellknown.CapabilityMetrics, - Target: &v1.InstallRequest{Cluster: &corev1.Reference{Id: "agent"}}, - }) - client.InstallCapability(context.Background(), &managementv1.CapabilityInstallRequest{ - Name: wellknown.CapabilityMetrics, - Target: &v1.InstallRequest{Cluster: &corev1.Reference{Id: "agent2"}}, - }) - - sloClient = sloapi.NewSLOClient(env.ManagementClientConn()) - adminClient = cortexadmin.NewCortexAdminClient(env.ManagementClientConn()) - Eventually(func() error { - stats, err := adminClient.AllUserStats(context.Background(), &emptypb.Empty{}) - if err != nil { - return err - } - agent1 := false - agent2 := false - for _, item := range stats.Items { - if item.UserID == "agent" { - if item.NumSeries > 0 { - agent1 = true - } - } - if item.UserID == "agent2" { - if item.NumSeries > 0 { - agent2 = true - } - } - } - if agent1 && agent2 { - return nil - } - return fmt.Errorf("waiting for metric data to be stored in cortex") - }, 30*time.Second, 1*time.Second).Should(Succeed()) - }) - - When("The instrumentation server starts", func() { - It("Should simulate events", func() { - Eventually(func() error { - if !canReachInstrumentationMetrics(instrumentationPort) { - return fmt.Errorf("cannot reach instrumentation server") - } - return nil - }, time.Second*10, time.Millisecond*500).Should(Succeed()) - Expect(instrumentationPort).NotTo(Equal(0)) - simulateGoodEvents("http-availability", instrumentationPort, 1000) - simulateBadEvents("http-availability", instrumentationPort, 1000) - }) - }) - - When("The SLO plugin starts, service discovery ", func() { - It("should be able to discover services from downstream", func() { - expectedNames := []string{"prometheus"} - resp, err := sloClient.ListServices(ctx, &sloapi.ListServicesRequest{ - Datasource: shared.MonitoringDatasource, - ClusterId: "agent", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.Items).NotTo(HaveLen(0)) - - for _, name := range expectedNames { - found := false - for _, svc := range resp.GetItems() { - if svc.GetServiceId() == name { - found = true - break - } - } - Expect(found).To(BeTrue()) - } - resp2, err := sloClient.ListServices(ctx, &sloapi.ListServicesRequest{ - Datasource: shared.MonitoringDatasource, - ClusterId: "agent2", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(resp2.Items).To(HaveLen(2)) - for _, name := range expectedNames { - found := false - for _, svc := range resp2.GetItems() { - if svc.GetServiceId() == name { - found = true - break - } - } - Expect(found).To(BeTrue()) - } - }) - - It("should be able to discover metrics from downstream", func() { - resp, err := sloClient.ListMetrics(ctx, &sloapi.ListMetricsRequest{ - Datasource: shared.MonitoringDatasource, - ClusterId: "agent", - ServiceId: "prometheus", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.GroupNameToMetrics).NotTo(HaveLen(0)) - for _, m := range resp.GroupNameToMetrics { - for _, metric := range m.Items { - Expect(metric.GetId()).NotTo(Equal("")) - } - } - }) - - It("Should be able to discover events from downstream", func() { - resp, err := sloClient.ListEvents(ctx, &sloapi.ListEventsRequest{ - Datasource: shared.MonitoringDatasource, - ServiceId: "prometheus", - ClusterId: "agent", - MetricId: "prometheus_http_requests_total", - }) - expected := []*sloapi.Event{ - { - Key: "code", - Vals: []string{ - "200", - }, - }, - { - Key: "handler", - Vals: []string{ - "/-/ready", - }, - }, - } - - Expect(err).NotTo(HaveOccurred()) - for _, ex := range expected { - foundKey := false - for _, output := range resp.Items { - if output.Key == ex.Key { - foundKey = true - for _, exval := range ex.Vals { - foundVal := false - for _, outval := range output.Vals { - if exval == outval { - foundVal = true - break - } - } - Expect(foundVal).To(BeTrue()) - } - break - } - } - Expect(foundKey).To(BeTrue()) - } - }) - }) - - When("CRUDing SLOs", func() { - It("Should error on invalid cluster id", func() { - _, err := sloClient.CreateSLO(ctx, &sloapi.CreateSLORequest{ - Slo: &sloapi.ServiceLevelObjective{ - Name: "testslo", - Datasource: shared.MonitoringDatasource, - ClusterId: "asdakjsdhkjashdjkahsdkjhakjsdhkjashdkj", - ServiceId: "prometheus", - GoodMetricName: "prometheus_http_requests_total", - TotalMetricName: "prometheus_http_requests_total", - GoodEvents: []*sloapi.Event{ - { - Key: "code", - Vals: []string{ - "200", - }, - }, - }, - TotalEvents: []*sloapi.Event{ - { - Key: "code", - Vals: []string{ - "200", - "500", - "503", - }, - }, - }, - SloPeriod: "1m", - BudgetingInterval: durationpb.New(time.Second * 1), - Target: &sloapi.Target{ - Value: 99.99, - }, - }, - }) - Expect(err).To(HaveOccurred()) - }) - - It("Should create SLOs", func() { - _, err := sloClient.CreateSLO(ctx, &sloapi.CreateSLORequest{ - Slo: &sloapi.ServiceLevelObjective{ - Name: "testslo", - Datasource: shared.MonitoringDatasource, - ClusterId: "agent", - ServiceId: "prometheus", - GoodMetricName: "prometheus_http_requests_total", - TotalMetricName: "prometheus_http_requests_total", - GoodEvents: []*sloapi.Event{ - { - Key: "code", - Vals: []string{ - "200", - }, - }, - }, - TotalEvents: []*sloapi.Event{ - { - Key: "code", - Vals: []string{ - "200", - "500", - "503", - }, - }, - }, - SloPeriod: "1m", - BudgetingInterval: durationpb.New(time.Second * 1), - Target: &sloapi.Target{ - Value: 99.99, - }, - }, - }) - Expect(err).NotTo(HaveOccurred()) - }) - - It("Should clone SLO", func() { - resp, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.Items).To(HaveLen(1)) - _, err = sloClient.CloneSLO(ctx, &corev1.Reference{Id: resp.Items[0].Id}) - Expect(err).NotTo(HaveOccurred()) - respAfter, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - Expect(respAfter.Items).To(HaveLen(2)) - }) - - It("Should error on update if provided a new cluster id", func() { - resp, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.Items).To(HaveLen(2)) - updateData := resp.Items[1] - updateData.SLO.Name = "test-slo-updated" - updateData.SLO.ClusterId = "asdkjasjkdhkajshdjkahsdjkhajkshdkjahsdjkhasjkdhkjasd" - _, err = sloClient.UpdateSLO(ctx, updateData) - Expect(err).To(HaveOccurred()) - - }) - - It("Should update SLOs", func() { - resp, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.Items).To(HaveLen(2)) - updateData := resp.Items[1] - updateData.SLO.Name = "test-slo-updated" - updateData.SLO.ClusterId = "agent2" - _, err = sloClient.UpdateSLO(ctx, updateData) - Expect(err).NotTo(HaveOccurred()) - respAfter, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - Expect(respAfter.Items).To(HaveLen(2)) - }) - - It("Should Get SLOs", func() { - resp, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.Items).To(HaveLen(2)) - _, err = sloClient.GetSLO(ctx, &corev1.Reference{Id: resp.Items[1].Id}) - Expect(err).NotTo(HaveOccurred()) - }) - - It("Should delete SLOs", func() { - resp, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.Items).To(HaveLen(2)) - _, err = sloClient.DeleteSLO(ctx, &corev1.Reference{Id: resp.Items[1].Id}) - Expect(err).NotTo(HaveOccurred()) - respAfter, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - Expect(respAfter.Items).To(HaveLen(1)) - }) - - It("Should get status for SLOs", func() { - respList, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - Expect(respList.Items).To(HaveLen(1)) - resp, err := sloClient.Status(ctx, &corev1.Reference{Id: respList.Items[0].Id}) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.State).To(Equal(sloapi.SLOStatusState_Creating)) - Eventually(func() sloapi.SLOStatusState { - resp, err := sloClient.Status(ctx, &corev1.Reference{Id: respList.Items[0].Id}) - Expect(err).NotTo(HaveOccurred()) - return resp.State - }, time.Second*60, time.Millisecond*500).Should(BeElementOf( - sloapi.SLOStatusState_Ok, - sloapi.SLOStatusState_PartialDataOk, - sloapi.SLOStatusState_Warning, - sloapi.SLOStatusState_Breaching, - )) - }) - - // FIXME: the following are disabled because they will take a long time to resolve correctly - - XIt("Should preview SLOs in a raw data format", func() { - Eventually(func() error { - respList, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - if err != nil { - return err - } - if len(respList.Items) != 1 { - return fmt.Errorf("expected number of SLOs to be 1") - } - resp, err := sloClient.Preview(ctx, &sloapi.CreateSLORequest{ - Slo: &sloapi.ServiceLevelObjective{ - Name: "testslo", - Datasource: shared.MonitoringDatasource, - ClusterId: "agent", - ServiceId: "prometheus", - GoodMetricName: "prometheus_http_requests_total", - TotalMetricName: "prometheus_http_requests_total", - GoodEvents: []*sloapi.Event{ - { - Key: "code", - Vals: []string{ - "200", - }, - }, - }, - TotalEvents: []*sloapi.Event{ - { - Key: "code", - Vals: []string{ - "200", - "500", - "503", - }, - }, - }, - SloPeriod: "1m", - BudgetingInterval: durationpb.New(time.Second * 1), - Target: &sloapi.Target{ - Value: 99.99, - }, - }, - }) - if err != nil { - return err - } - if len(resp.PlotVector.Items) == 0 { - return fmt.Errorf("expected plot vector items to not be empty") - } - hasData := false - for _, y := range resp.PlotVector.Items { - if y.Sli > 0 { - hasData = true - break - } - } - if !hasData { - return fmt.Errorf("Expected plot vector to have some non-zero data") - } - last := resp.PlotVector.Items[len(resp.PlotVector.Items)-1] - if last.Sli == 0 { - return fmt.Errorf("SLI should not be zero") - } - if last.Sli > 100 { - return fmt.Errorf("SLI should not be greater than 100") - } - - if len(resp.PlotVector.Windows) == 0 { - return fmt.Errorf("expected an alert to be firing based on event simulation") - } - - // both alerts should be firing based on the number of bad events detected - hasSevere, hasCritical := false, false - for _, w := range resp.PlotVector.Windows { - if w.Severity == "severe" { - hasSevere = true - } - if w.Severity == "critical" { - hasCritical = true - } - } - if !hasSevere || !hasCritical { - return fmt.Errorf("expected both severe and critical alerts to be firing") - } - return nil - }, time.Second*120, time.Second).Should(Succeed()) - - }) - - XSpecify("Creating an SLO for the service that should be alerting", func() { - failingSloId, err := sloClient.CreateSLO(ctx, &sloapi.CreateSLORequest{ - Slo: &sloapi.ServiceLevelObjective{ - Name: "testslo", - Datasource: shared.MonitoringDatasource, - ClusterId: "agent", - ServiceId: "MyServer", - GoodMetricName: "http_request_duration_seconds_count", - TotalMetricName: "http_request_duration_seconds_count", - GoodEvents: []*sloapi.Event{ - { - Key: "code", - Vals: []string{ - "200", - }, - }, - }, - TotalEvents: []*sloapi.Event{ - { - Key: "code", - Vals: []string{ - "200", - "500", - "501", - "502", - "503", - }, - }, - }, - SloPeriod: "1m", - BudgetingInterval: durationpb.New(time.Second * 1), - Target: &sloapi.Target{ - Value: 99.99, - }, - }, - }) - Expect(err).NotTo(HaveOccurred()) - Eventually(func() sloapi.SLOStatusState { - resp, err := sloClient.Status(ctx, &corev1.Reference{Id: failingSloId.Id}) - Expect(err).NotTo(HaveOccurred()) - return resp.State - }, time.Minute, time.Second*1).Should(BeElementOf(sloapi.SLOStatusState_Warning, sloapi.SLOStatusState_Breaching)) - }) - - Specify("Multi Cluster Clone should clone to valid targets", func() { - slos, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).To(Succeed()) - Expect(slos.Items).NotTo(HaveLen(0)) - toCloneId := slos.Items[0].Id - var agentsCancel []context.CancelFunc - var agentClusterIds []*corev1.Reference - for i := 0; i < 10; i++ { - id := fmt.Sprintf("agent-%d", i) - ctxCa, cancelFunc := context.WithCancel(ctx) - _, errC := env.StartAgent( - id, - token, - []string{info.Chain[len(info.Chain)-1].Fingerprint}, - test.WithContext(ctxCa), - ) - env.SetPrometheusNodeConfigOverride(id, test.NewOverridePrometheusConfig( - "slo/prometheus/config.yaml", - []test.PrometheusJob{ - { - JobName: query.MockTestServerName, - ScrapePort: instrumentationPort, - }, - }), - ) - Eventually(errC).Should(Receive(BeNil())) - - _, err := client.InstallCapability(ctx, &managementv1.CapabilityInstallRequest{ - Name: wellknown.CapabilityMetrics, - Target: &v1.InstallRequest{Cluster: &corev1.Reference{Id: id}}, - }) - Expect(err).NotTo(HaveOccurred()) - - agentsCancel = append(agentsCancel, cancelFunc) - agentClusterIds = append(agentClusterIds, &corev1.Reference{Id: id}) - } - - Eventually(func() error { - stats, err := adminClient.AllUserStats(context.Background(), &emptypb.Empty{}) - if err != nil { - return err - } - for _, expectedItem := range agentClusterIds { - found := false - for _, item := range stats.Items { - if item.UserID == expectedItem.Id { - if item.NumSeries > 0 { - found = true - } - } - } - if !found { - return fmt.Errorf("Waiting for metric data for cluster %s", expectedItem.Id) - } - } - return nil - - }, 30*time.Second, 1*time.Second).Should(Succeed()) - - failures, err := sloClient.CloneToClusters(ctx, &sloapi.MultiClusterSLO{ - CloneId: &corev1.Reference{Id: toCloneId}, - Clusters: agentClusterIds, - }) - Expect(err).To(Succeed()) - Expect(failures.Failures).To(BeEmpty()) - - for _, cancelFunc := range agentsCancel { - cancelFunc() - } - }) - }) -}) diff --git a/test/plugins/slo/test_helpers_test.go b/test/plugins/slo/test_helpers_test.go deleted file mode 100644 index 857060647f..0000000000 --- a/test/plugins/slo/test_helpers_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package plugins_test - -import ( - "context" - "fmt" - "sync" - "time" - - . "github.com/onsi/gomega" - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - "github.com/rancher/opni/plugins/slo/pkg/slo" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func sloCortexGroupsToCheck(groupName string) []string { - return []string{ - groupName + slo.RecordingRuleSuffix, - groupName + slo.MetadataRuleSuffix, - groupName + slo.AlertRuleSuffix, - } -} - -func expectSLOGroupToExist(ctx context.Context, adminClient cortexadmin.CortexAdminClient, tenant string, groupName string) { - var anyError error - var wg sync.WaitGroup - groupsToCheck := sloCortexGroupsToCheck(groupName) - wg.Add(len(groupsToCheck)) - - for _, group := range groupsToCheck { - groupToCheck := group - go func() { - defer wg.Done() - if err := expectRuleGroupToExist(ctx, adminClient, tenant, groupToCheck); err != nil { - anyError = err - } - }() - } - wg.Wait() - Expect(anyError).Should(BeNil()) -} - -func expectSLOGroupNotToExist(ctx context.Context, adminClient cortexadmin.CortexAdminClient, tenant string, groupName string) { - var anyError error - var wg sync.WaitGroup - groupsToCheck := sloCortexGroupsToCheck(groupName) - wg.Add(len(groupsToCheck)) - - for _, group := range groupsToCheck { - groupToCheck := group - go func() { - defer wg.Done() - if err := expectRuleGroupNotToExist(ctx, adminClient, tenant, groupToCheck); err != nil { - anyError = err - } - }() - } - wg.Wait() - Expect(anyError).Should(BeNil()) -} - -// potentially "long" running function, call asynchronously -func expectRuleGroupToExist(ctx context.Context, adminClient cortexadmin.CortexAdminClient, tenant string, groupName string) error { - for i := 0; i < 10; i++ { - resp, err := adminClient.GetRule(ctx, &cortexadmin.GetRuleRequest{ - ClusterId: tenant, - Namespace: "test", - GroupName: groupName, - }) - if err == nil { - Expect(resp.Data).To(Not(BeNil())) - return nil - } - time.Sleep(1) - } - return fmt.Errorf("Rule %s should exist, but doesn't", groupName) -} - -// potentially "long" running function, call asynchronously -func expectRuleGroupNotToExist(ctx context.Context, adminClient cortexadmin.CortexAdminClient, tenant string, groupName string) error { - for i := 0; i < 10; i++ { - _, err := adminClient.GetRule(ctx, &cortexadmin.GetRuleRequest{ - ClusterId: tenant, - Namespace: "test", - GroupName: groupName, - }) - if err != nil { - Expect(status.Code(err)).To(Equal(codes.NotFound)) - return nil - } - - time.Sleep(1) - } - return fmt.Errorf("Rule %s still exists, but shouldn't", groupName) -} From ec2d9d9b74f8bf6ee7cb11881d809850e210b02a Mon Sep 17 00:00:00 2001 From: Alexandre Lamarre Date: Thu, 14 Sep 2023 11:40:58 -0400 Subject: [PATCH 2/7] rework alerting integration test setup --- test/plugins/alerting/alerting_suite_test.go | 26 +- test/plugins/alerting/alerting_test.go | 1454 +++++++++--------- test/plugins/alerting/routing_test.go | 405 ++--- test/plugins/alerting/slo_test.go | 1 + 4 files changed, 927 insertions(+), 959 deletions(-) diff --git a/test/plugins/alerting/alerting_suite_test.go b/test/plugins/alerting/alerting_suite_test.go index 4c29cdc187..acab5e22be 100644 --- a/test/plugins/alerting/alerting_suite_test.go +++ b/test/plugins/alerting/alerting_suite_test.go @@ -8,9 +8,7 @@ import ( . "github.com/onsi/gomega" "github.com/prometheus/common/model" "github.com/rancher/opni/pkg/alerting/drivers/routing" - "github.com/rancher/opni/pkg/test" _ "github.com/rancher/opni/pkg/test/setup" - "github.com/rancher/opni/pkg/test/testruntime" _ "github.com/rancher/opni/plugins/alerting/test" _ "github.com/rancher/opni/plugins/metrics/test" "github.com/samber/lo" @@ -40,16 +38,16 @@ func TestAlerting(t *testing.T) { RunSpecs(t, "Alerting Suite") } -var env *test.Environment -var tmpConfigDir string +// var env *test.Environment +// var tmpConfigDir string -var _ = BeforeSuite(func() { - testruntime.IfIntegration(func() { - env = &test.Environment{} - Expect(env).NotTo(BeNil()) - Expect(env.Start()).To(Succeed()) - DeferCleanup(env.Stop) - tmpConfigDir = env.GenerateNewTempDirectory("alertmanager-config") - Expect(tmpConfigDir).NotTo(Equal("")) - }) -}) +// var _ = BeforeSuite(func() { +// testruntime.IfIntegration(func() { +// env = &test.Environment{} +// Expect(env).NotTo(BeNil()) +// Expect(env.Start()).To(Succeed()) +// DeferCleanup(env.Stop) +// tmpConfigDir = env.GenerateNewTempDirectory("alertmanager-config") +// Expect(tmpConfigDir).NotTo(Equal("")) +// }) +// }) diff --git a/test/plugins/alerting/alerting_test.go b/test/plugins/alerting/alerting_test.go index eef4aee75c..58f85541e8 100644 --- a/test/plugins/alerting/alerting_test.go +++ b/test/plugins/alerting/alerting_test.go @@ -22,7 +22,6 @@ import ( managementv1 "github.com/rancher/opni/pkg/apis/management/v1" "github.com/rancher/opni/pkg/test" "github.com/rancher/opni/pkg/test/alerting" - "github.com/rancher/opni/pkg/test/testruntime" "github.com/rancher/opni/plugins/alerting/apis/alertops" "github.com/samber/lo" "golang.org/x/exp/slices" @@ -39,51 +38,6 @@ const ( numNotificationServers = 2 ) -func init() { - testruntime.IfIntegration(func() { - BuildAlertingClusterIntegrationTests([]*alertops.ClusterConfiguration{ - // { - // NumReplicas: 1, - // ResourceLimits: &alertops.ResourceLimitSpec{ - // Cpu: "100m", - // Memory: "100Mi", - // Storage: "5Gi", - // }, - // ClusterSettleTimeout: "1m", - // ClusterGossipInterval: "1m", - // ClusterPushPullInterval: "1m", - // }, - { - NumReplicas: 3, - ResourceLimits: &alertops.ResourceLimitSpec{ - Cpu: "100m", - Memory: "100Mi", - Storage: "5Gi", - }, - ClusterSettleTimeout: "1m", - ClusterGossipInterval: "1m", - ClusterPushPullInterval: "1m", - }, - }, - func() alertops.AlertingAdminClient { - return alertops.NewAlertingAdminClient(env.ManagementClientConn()) - }, - func() alertingv1.AlertConditionsClient { - return env.NewAlertConditionsClient() - }, - func() alertingv1.AlertEndpointsClient { - return env.NewAlertEndpointsClient() - }, - func() alertingv1.AlertNotificationsClient { - return env.NewAlertNotificationsClient() - }, - func() managementv1.ManagementClient { - return env.NewManagementClient() - }, - ) - }) -} - type agentWithContext struct { id string context.Context @@ -91,829 +45,843 @@ type agentWithContext struct { port int } -func BuildAlertingClusterIntegrationTests( - clusterConfigurations []*alertops.ClusterConfiguration, - alertingAdminConstructor func() alertops.AlertingAdminClient, - alertingConditionsConstructor func() alertingv1.AlertConditionsClient, - alertingEndpointsConstructor func() alertingv1.AlertEndpointsClient, - alertingNotificationsConstructor func() alertingv1.AlertNotificationsClient, - mgmtClientConstructor func() managementv1.ManagementClient, -) bool { - return Describe("Alerting Cluster Integration tests", Ordered, Label("integration"), func() { - var alertClusterClient alertops.AlertingAdminClient - var alertEndpointsClient alertingv1.AlertEndpointsClient - var alertConditionsClient alertingv1.AlertConditionsClient - var alertNotificationsClient alertingv1.AlertNotificationsClient - var mgmtClient managementv1.ManagementClient - var numAgents int - - // contains agent id and other useful metadata and functions - var agents []*agentWithContext - // physical servers that receive opni alerting notifications - var servers []*alerting.MockIntegrationWebhookServer - // physical servers that receive all opni alerting notifications - var notificationServers []*alerting.MockIntegrationWebhookServer - // conditionsIds => endopintIds - expectedRouting := map[string]*alertingv1.ConditionReferenceList{} - // maps condition ids where agents are disconnect to their webhook ids - involvedDisconnects := map[string][]string{} - - var httpProxyClient *http.Client - When("Installing the Alerting Cluster", func() { - BeforeAll(func() { - alertClusterClient = alertingAdminConstructor() - alertEndpointsClient = alertingEndpointsConstructor() - alertConditionsClient = alertingConditionsConstructor() - alertNotificationsClient = alertingNotificationsConstructor() - mgmtClient = mgmtClientConstructor() - numAgents = 5 - tlsConfig := env.GatewayClientTLSConfig() - httpProxyClient = &http.Client{ - Transport: &http.Transport{ - DialTLS: func(network, addr string) (net.Conn, error) { - conn, err := tls.Dial(network, addr, tlsConfig) - return conn, err - }, +var _ = Describe("Alerting Cluster Integration tests", Ordered, Label("integration"), func() { + var env *test.Environment + + var clusterConfigurations = []*alertops.ClusterConfiguration{ + { + NumReplicas: 3, + ResourceLimits: &alertops.ResourceLimitSpec{ + Cpu: "100m", + Memory: "100Mi", + Storage: "5Gi", + }, + ClusterSettleTimeout: "1m", + ClusterGossipInterval: "1m", + ClusterPushPullInterval: "1m", + }, + } + + var alertClusterClient alertops.AlertingAdminClient + var alertEndpointsClient alertingv1.AlertEndpointsClient + var alertConditionsClient alertingv1.AlertConditionsClient + var alertNotificationsClient alertingv1.AlertNotificationsClient + var mgmtClient managementv1.ManagementClient + var numAgents int + + // contains agent id and other useful metadata and functions + var agents []*agentWithContext + // physical servers that receive opni alerting notifications + var servers []*alerting.MockIntegrationWebhookServer + // physical servers that receive all opni alerting notifications + var notificationServers []*alerting.MockIntegrationWebhookServer + // conditionsIds => endopintIds + expectedRouting := map[string]*alertingv1.ConditionReferenceList{} + // maps condition ids where agents are disconnect to their webhook ids + involvedDisconnects := map[string][]string{} + + var httpProxyClient *http.Client + When("Installing the Alerting Cluster", func() { + BeforeAll(func() { + env = &test.Environment{} + Expect(env).NotTo(BeNil()) + Expect(env.Start()).To(Succeed()) + DeferCleanup(env.Stop) + tmpConfigDir := env.GenerateNewTempDirectory("alertmanager-config") + Expect(tmpConfigDir).NotTo(Equal("")) + + alertClusterClient = alertops.NewAlertingAdminClient(env.ManagementClientConn()) + alertEndpointsClient = env.NewAlertEndpointsClient() + alertConditionsClient = env.NewAlertConditionsClient() + alertNotificationsClient = env.NewAlertNotificationsClient() + mgmtClient = env.NewManagementClient() + numAgents = 5 + tlsConfig := env.GatewayClientTLSConfig() + httpProxyClient = &http.Client{ + Transport: &http.Transport{ + DialTLS: func(network, addr string) (net.Conn, error) { + conn, err := tls.Dial(network, addr, tlsConfig) + return conn, err }, - } - }) + }, + } + }) - AfterEach(func() { - status, err := alertClusterClient.GetClusterStatus(env.Context(), &emptypb.Empty{}) + AfterEach(func() { + status, err := alertClusterClient.GetClusterStatus(env.Context(), &emptypb.Empty{}) + Expect(err).To(BeNil()) + installed := status.State != alertops.InstallState_NotInstalled && status.State != alertops.InstallState_Uninstalling + if installed { + info, err := alertClusterClient.Info(env.Context(), &emptypb.Empty{}) Expect(err).To(BeNil()) - installed := status.State != alertops.InstallState_NotInstalled && status.State != alertops.InstallState_Uninstalling - if installed { - info, err := alertClusterClient.Info(env.Context(), &emptypb.Empty{}) - Expect(err).To(BeNil()) - Expect(info).NotTo(BeNil()) - Expect(info.CurSyncId).NotTo(BeEmpty()) - ts := timestamppb.Now() - for _, comp := range info.Components { - Expect(comp.LastHandshake.AsTime()).To(BeTemporally("<", ts.AsTime())) - if comp.ConnectInfo.GetState() != alertops.SyncState_Synced { - if comp.ConnectInfo.SyncId != info.CurSyncId { - Expect(comp.LastHandshake.AsTime()).To(BeTemporally(">", ts.AsTime().Add(15*(-time.Second))), "sync is stuttering") - } + Expect(info).NotTo(BeNil()) + Expect(info.CurSyncId).NotTo(BeEmpty()) + ts := timestamppb.Now() + for _, comp := range info.Components { + Expect(comp.LastHandshake.AsTime()).To(BeTemporally("<", ts.AsTime())) + if comp.ConnectInfo.GetState() != alertops.SyncState_Synced { + if comp.ConnectInfo.SyncId != info.CurSyncId { + Expect(comp.LastHandshake.AsTime()).To(BeTemporally(">", ts.AsTime().Add(15*(-time.Second))), "sync is stuttering") } } } - }) - - for _, clusterConf := range clusterConfigurations { - It("should install the alerting cluster", func() { - _, err := alertClusterClient.InstallCluster(env.Context(), &emptypb.Empty{}) - Expect(err).To(BeNil()) + } + }) - Eventually(func() error { - status, err := alertClusterClient.GetClusterStatus(env.Context(), &emptypb.Empty{}) - if err != nil { - return err - } - if status.State != alertops.InstallState_Installed { - return fmt.Errorf("alerting cluster install state is %s", status.State.String()) - } - return nil - }, time.Second*5, time.Millisecond*200).Should(Succeed()) - }) + for _, clusterConf := range clusterConfigurations { + It("should install the alerting cluster", func() { + _, err := alertClusterClient.InstallCluster(env.Context(), &emptypb.Empty{}) + Expect(err).To(BeNil()) - It("should apply the configuration", func() { - _, err := alertClusterClient.ConfigureCluster(env.Context(), clusterConf) + Eventually(func() error { + status, err := alertClusterClient.GetClusterStatus(env.Context(), &emptypb.Empty{}) if err != nil { - if s, ok := status.FromError(err); ok { // conflict is ok if using default config - Expect(s.Code()).To(Equal(codes.FailedPrecondition)) - } + return err } - Expect(err).To(BeNil()) - Eventually(func() error { - status, err := alertClusterClient.GetClusterStatus(env.Context(), &emptypb.Empty{}) - if err != nil { - return err - } - if status.State != alertops.InstallState_Installed { - return fmt.Errorf("alerting cluster install state is %s", status.State.String()) - } - return nil - }, time.Second*5, time.Millisecond*200).Should(Succeed()) + if status.State != alertops.InstallState_Installed { + return fmt.Errorf("alerting cluster install state is %s", status.State.String()) + } + return nil + }, time.Second*5, time.Millisecond*200).Should(Succeed()) + }) - Eventually(func() error { - getConf, err := alertClusterClient.GetClusterConfiguration(env.Context(), &emptypb.Empty{}) - if !proto.Equal(getConf, clusterConf) { - return fmt.Errorf("cluster config not equal : not applied") - } + It("should apply the configuration", func() { + _, err := alertClusterClient.ConfigureCluster(env.Context(), clusterConf) + if err != nil { + if s, ok := status.FromError(err); ok { // conflict is ok if using default config + Expect(s.Code()).To(Equal(codes.FailedPrecondition)) + } + } + Expect(err).To(BeNil()) + Eventually(func() error { + status, err := alertClusterClient.GetClusterStatus(env.Context(), &emptypb.Empty{}) + if err != nil { return err - }, time.Second*5, time.Millisecond*200).Should(Succeed()) - }) + } + if status.State != alertops.InstallState_Installed { + return fmt.Errorf("alerting cluster install state is %s", status.State.String()) + } + return nil + }, time.Second*5, time.Millisecond*200).Should(Succeed()) + + Eventually(func() error { + getConf, err := alertClusterClient.GetClusterConfiguration(env.Context(), &emptypb.Empty{}) + if !proto.Equal(getConf, clusterConf) { + return fmt.Errorf("cluster config not equal : not applied") + } + return err + }, time.Second*5, time.Millisecond*200).Should(Succeed()) + }) - Specify("the alerting plugin components should be running and healthy", func() { - alertingReadinessProbe := fmt.Sprintf("https://%s/plugin_alerting/ready", env.GatewayConfig().Spec.HTTPListenAddress) - alertingHealthProbe := fmt.Sprintf("https://%s/plugin_alerting/healthy", env.GatewayConfig().Spec.HTTPListenAddress) + Specify("the alerting plugin components should be running and healthy", func() { + alertingReadinessProbe := fmt.Sprintf("https://%s/plugin_alerting/ready", env.GatewayConfig().Spec.HTTPListenAddress) + alertingHealthProbe := fmt.Sprintf("https://%s/plugin_alerting/healthy", env.GatewayConfig().Spec.HTTPListenAddress) - Eventually(func() error { - reqReady, err := http.NewRequestWithContext(env.Context(), http.MethodGet, alertingReadinessProbe, nil) - if err != nil { - return err - } - reqHealthy, err := http.NewRequestWithContext(env.Context(), http.MethodGet, alertingHealthProbe, nil) - if err != nil { - return err - } - respReady, err := httpProxyClient.Do(reqReady) - if err != nil { - return err - } - defer respReady.Body.Close() - if respReady.StatusCode != http.StatusOK { - return fmt.Errorf("alerting plugin not yet ready %d, %s", respReady.StatusCode, respReady.Status) - } - respHealthy, err := httpProxyClient.Do(reqHealthy) + Eventually(func() error { + reqReady, err := http.NewRequestWithContext(env.Context(), http.MethodGet, alertingReadinessProbe, nil) + if err != nil { + return err + } + reqHealthy, err := http.NewRequestWithContext(env.Context(), http.MethodGet, alertingHealthProbe, nil) + if err != nil { + return err + } + respReady, err := httpProxyClient.Do(reqReady) + if err != nil { + return err + } + defer respReady.Body.Close() + if respReady.StatusCode != http.StatusOK { + return fmt.Errorf("alerting plugin not yet ready %d, %s", respReady.StatusCode, respReady.Status) + } + respHealthy, err := httpProxyClient.Do(reqHealthy) + if err != nil { + return err + } + defer respHealthy.Body.Close() + if respHealthy.StatusCode != http.StatusOK { if err != nil { - return err - } - defer respHealthy.Body.Close() - if respHealthy.StatusCode != http.StatusOK { - if err != nil { - return fmt.Errorf("alerting plugin unhealthy %d, %s", respHealthy.StatusCode, respHealthy.Status) - } + return fmt.Errorf("alerting plugin unhealthy %d, %s", respHealthy.StatusCode, respHealthy.Status) } - return nil - }, time.Second*30, time.Second).Should(Succeed()) - }) - - It("should be able to create some endpoints", func() { - servers = alerting.CreateWebhookServer(env, numServers) - - for _, server := range servers { - ref, err := alertEndpointsClient.CreateAlertEndpoint(env.Context(), server.Endpoint()) - Expect(err).To(Succeed()) - server.EndpointId = ref.Id } + return nil + }, time.Second*30, time.Second).Should(Succeed()) + }) - By("verifying they are externally persisted") + It("should be able to create some endpoints", func() { + servers = alerting.CreateWebhookServer(env, numServers) - endpList, err := alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) + for _, server := range servers { + ref, err := alertEndpointsClient.CreateAlertEndpoint(env.Context(), server.Endpoint()) Expect(err).To(Succeed()) - Expect(endpList.Items).To(HaveLen(numServers)) + server.EndpointId = ref.Id + } - By("verifying they are reachable") + By("verifying they are externally persisted") - for _, endp := range endpList.GetItems() { - Expect(endp.GetId()).NotTo(BeNil()) - _, err := alertNotificationsClient.TestAlertEndpoint(env.Context(), endp.GetId()) - Expect(err).To(Succeed()) - } - maxSuccesses := 0 - Eventually(func() error { - success := 0 - errs := []error{} - for _, server := range servers { - if len(server.GetBuffer()) == 0 { - if success > maxSuccesses { - maxSuccesses = success - } - errs = append(errs, fmt.Errorf("server %v did not receive any alerts", server.Endpoint())) - } else { - success++ - } - } - if len(errs) > 0 { - return errors.Join(errs...) - } - return nil - }, time.Second*15, time.Millisecond*100).Should(Succeed(), fmt.Sprintf("only %d/%d servers received alerts", maxSuccesses, numServers)) + endpList, err := alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) + Expect(err).To(Succeed()) + Expect(endpList.Items).To(HaveLen(numServers)) + + By("verifying they are reachable") + for _, endp := range endpList.GetItems() { + Expect(endp.GetId()).NotTo(BeNil()) + _, err := alertNotificationsClient.TestAlertEndpoint(env.Context(), endp.GetId()) + Expect(err).To(Succeed()) + } + maxSuccesses := 0 + Eventually(func() error { + success := 0 + errs := []error{} for _, server := range servers { - server.ClearBuffer() + if len(server.GetBuffer()) == 0 { + if success > maxSuccesses { + maxSuccesses = success + } + errs = append(errs, fmt.Errorf("server %v did not receive any alerts", server.Endpoint())) + } else { + success++ + } } - }) + if len(errs) > 0 { + return errors.Join(errs...) + } + return nil + }, time.Second*15, time.Millisecond*100).Should(Succeed(), fmt.Sprintf("only %d/%d servers received alerts", maxSuccesses, numServers)) - It("should create some default conditions when bootstrapping agents", func() { - By("expecting to have no initial conditions") - Eventually(func() error { - condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) - if err != nil { - return err - } - if len(condList.Items) != 0 { - return fmt.Errorf("expected 0 conditions, got %d", len(condList.Items)) - } - return nil - }, time.Second*30, time.Millisecond*200).Should(Succeed()) + for _, server := range servers { + server.ClearBuffer() + } + }) - By(fmt.Sprintf("bootstrapping %d agents", numAgents)) - certsInfo, err := mgmtClient.CertsInfo(context.Background(), &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - fingerprint := certsInfo.Chain[len(certsInfo.Chain)-1].Fingerprint - Expect(fingerprint).NotTo(BeEmpty()) + It("should create some default conditions when bootstrapping agents", func() { + By("expecting to have no initial conditions") + Eventually(func() error { + condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) + if err != nil { + return err + } + if len(condList.Items) != 0 { + return fmt.Errorf("expected 0 conditions, got %d", len(condList.Items)) + } + return nil + }, time.Second*30, time.Millisecond*200).Should(Succeed()) - token, err := mgmtClient.CreateBootstrapToken(context.Background(), &managementv1.CreateBootstrapTokenRequest{ - Ttl: durationpb.New(1 * time.Hour), + By(fmt.Sprintf("bootstrapping %d agents", numAgents)) + certsInfo, err := mgmtClient.CertsInfo(context.Background(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + fingerprint := certsInfo.Chain[len(certsInfo.Chain)-1].Fingerprint + Expect(fingerprint).NotTo(BeEmpty()) + + token, err := mgmtClient.CreateBootstrapToken(context.Background(), &managementv1.CreateBootstrapTokenRequest{ + Ttl: durationpb.New(1 * time.Hour), + }) + Expect(err).NotTo(HaveOccurred()) + agentIdFunc := func(i int) string { + return fmt.Sprintf("agent-%d-%s", i, uuid.New().String()) + } + agents = []*agentWithContext{} + for i := 0; i < numAgents; i++ { + ctxCa, ca := context.WithCancel(env.Context()) + id := agentIdFunc(i) + port, errC := env.StartAgent(id, token, []string{fingerprint}, test.WithContext(ctxCa)) + Eventually(errC).Should(Receive(BeNil())) + agents = append(agents, &agentWithContext{ + port: port, + CancelFunc: ca, + Context: ctxCa, + id: id, }) - Expect(err).NotTo(HaveOccurred()) - agentIdFunc := func(i int) string { - return fmt.Sprintf("agent-%d-%s", i, uuid.New().String()) - } - agents = []*agentWithContext{} - for i := 0; i < numAgents; i++ { - ctxCa, ca := context.WithCancel(env.Context()) - id := agentIdFunc(i) - port, errC := env.StartAgent(id, token, []string{fingerprint}, test.WithContext(ctxCa)) - Eventually(errC).Should(Receive(BeNil())) - agents = append(agents, &agentWithContext{ - port: port, - CancelFunc: ca, - Context: ctxCa, - id: id, - }) + } + By("verifying that there are default conditions") + Eventually(func() error { + condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) + if err != nil { + return err + } + if len(condList.Items) != numAgents*2 { + return fmt.Errorf("expected %d conditions, got %d", numAgents*2, len(condList.Items)) } - By("verifying that there are default conditions") + return nil + }, time.Second*5, time.Millisecond*200).Should(Succeed()) + + By("verifying that only the default group exists") + groupList, err := alertConditionsClient.ListAlertConditionGroups(env.Context(), &emptypb.Empty{}) + Expect(err).To(Succeed()) + Expect(groupList.Items).To(HaveLen(1)) + Expect(groupList.Items[0].Id).To(Equal("")) + }) + + It("shoud list conditions by given filters", func() { + for _, agent := range agents { Eventually(func() error { - condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) + filteredByCluster, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ + Clusters: []string{agent.id}, + }) if err != nil { return err } - if len(condList.Items) != numAgents*2 { - return fmt.Errorf("expected %d conditions, got %d", numAgents*2, len(condList.Items)) + if len(filteredByCluster.Items) != 2 { + return fmt.Errorf("expected 2 conditions, got %d", len(filteredByCluster.Items)) } return nil - }, time.Second*5, time.Millisecond*200).Should(Succeed()) + }, time.Second).Should(Succeed()) - By("verifying that only the default group exists") - groupList, err := alertConditionsClient.ListAlertConditionGroups(env.Context(), &emptypb.Empty{}) - Expect(err).To(Succeed()) - Expect(groupList.Items).To(HaveLen(1)) - Expect(groupList.Items[0].Id).To(Equal("")) - }) - - It("shoud list conditions by given filters", func() { - for _, agent := range agents { - Eventually(func() error { - filteredByCluster, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ - Clusters: []string{agent.id}, - }) - if err != nil { - return err - } - if len(filteredByCluster.Items) != 2 { - return fmt.Errorf("expected 2 conditions, got %d", len(filteredByCluster.Items)) - } - return nil - }, time.Second).Should(Succeed()) - - } + } - By("verifying all the agents conditions are critical") + By("verifying all the agents conditions are critical") - filterList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ - Severities: []alertingv1.OpniSeverity{ - alertingv1.OpniSeverity_Warning, - alertingv1.OpniSeverity_Error, - alertingv1.OpniSeverity_Info, - }, - }) - Expect(err).To(Succeed()) - Expect(filterList.Items).To(HaveLen(0)) + filterList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ + Severities: []alertingv1.OpniSeverity{ + alertingv1.OpniSeverity_Warning, + alertingv1.OpniSeverity_Error, + alertingv1.OpniSeverity_Info, + }, + }) + Expect(err).To(Succeed()) + Expect(filterList.Items).To(HaveLen(0)) - By("verifying we have an equal number of disconnect and capability unhealthy") + By("verifying we have an equal number of disconnect and capability unhealthy") - disconnectList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ + disconnectList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ + AlertTypes: []alertingv1.AlertType{ + alertingv1.AlertType_System, + }, + }) + Expect(err).To(Succeed()) + disconnectStatusList, err := alertConditionsClient.ListAlertConditionsWithStatus(env.Context(), &alertingv1.ListStatusRequest{ + ItemFilter: &alertingv1.ListAlertConditionRequest{ AlertTypes: []alertingv1.AlertType{ alertingv1.AlertType_System, }, - }) - Expect(err).To(Succeed()) - disconnectStatusList, err := alertConditionsClient.ListAlertConditionsWithStatus(env.Context(), &alertingv1.ListStatusRequest{ - ItemFilter: &alertingv1.ListAlertConditionRequest{ - AlertTypes: []alertingv1.AlertType{ - alertingv1.AlertType_System, - }, - }, - }) - Expect(err).To(Succeed()) + }, + }) + Expect(err).To(Succeed()) + + capabilityList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ + AlertTypes: []alertingv1.AlertType{ + alertingv1.AlertType_DownstreamCapability, + }, + }) + Expect(err).To(Succeed()) - capabilityList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ + capabilityStatusList, err := alertConditionsClient.ListAlertConditionsWithStatus(env.Context(), &alertingv1.ListStatusRequest{ + ItemFilter: &alertingv1.ListAlertConditionRequest{ AlertTypes: []alertingv1.AlertType{ alertingv1.AlertType_DownstreamCapability, }, - }) - Expect(err).To(Succeed()) - - capabilityStatusList, err := alertConditionsClient.ListAlertConditionsWithStatus(env.Context(), &alertingv1.ListStatusRequest{ - ItemFilter: &alertingv1.ListAlertConditionRequest{ - AlertTypes: []alertingv1.AlertType{ - alertingv1.AlertType_DownstreamCapability, - }, - }, - }) - - Expect(err).To(Succeed()) - Expect(capabilityList.Items).To(HaveLen(len(disconnectList.Items))) - Expect(capabilityStatusList.GetAlertConditions()).To(HaveLen(len(disconnectStatusList.GetAlertConditions()))) + }, }) - It("should be able to attach endpoints to conditions", func() { - By("attaching a sample of random endpoints to default agent conditions") - condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) + Expect(err).To(Succeed()) + Expect(capabilityList.Items).To(HaveLen(len(disconnectList.Items))) + Expect(capabilityStatusList.GetAlertConditions()).To(HaveLen(len(disconnectStatusList.GetAlertConditions()))) + }) + + It("should be able to attach endpoints to conditions", func() { + By("attaching a sample of random endpoints to default agent conditions") + condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) + Expect(err).To(Succeed()) + endpList, err := alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) + for _, cond := range condList.Items { Expect(err).To(Succeed()) - endpList, err := alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) - for _, cond := range condList.Items { - Expect(err).To(Succeed()) - endps := lo.Map( - lo.Samples(endpList.Items, 1+rand.Intn(len(endpList.Items)-1)), - func(a *alertingv1.AlertEndpointWithId, _ int) *alertingv1.AttachedEndpoint { - return &alertingv1.AttachedEndpoint{ - EndpointId: a.GetId().Id, - } - }) + endps := lo.Map( + lo.Samples(endpList.Items, 1+rand.Intn(len(endpList.Items)-1)), + func(a *alertingv1.AlertEndpointWithId, _ int) *alertingv1.AttachedEndpoint { + return &alertingv1.AttachedEndpoint{ + EndpointId: a.GetId().Id, + } + }) - if cond.GetAlertCondition().GetAlertType().GetSystem() != nil { - for _, endp := range endps { - if _, ok := expectedRouting[endp.EndpointId]; !ok { - expectedRouting[endp.EndpointId] = &alertingv1.ConditionReferenceList{ - Items: []*alertingv1.ConditionReference{}, - } + if cond.GetAlertCondition().GetAlertType().GetSystem() != nil { + for _, endp := range endps { + if _, ok := expectedRouting[endp.EndpointId]; !ok { + expectedRouting[endp.EndpointId] = &alertingv1.ConditionReferenceList{ + Items: []*alertingv1.ConditionReference{}, } - expectedRouting[endp.EndpointId].Items = append(expectedRouting[endp.EndpointId].Items, cond.GetId()) } - cond.AlertCondition.AttachedEndpoints = &alertingv1.AttachedEndpoints{ - Items: endps, - InitialDelay: durationpb.New(time.Second * 1), - ThrottlingDuration: durationpb.New(time.Second * 1), - Details: &alertingv1.EndpointImplementation{ - Title: "disconnected agent", - Body: "agent %s is disconnected", - }, - } - _, err = alertConditionsClient.UpdateAlertCondition(env.Context(), &alertingv1.UpdateAlertConditionRequest{ - Id: cond.GetId(), - UpdateAlert: cond.AlertCondition, - }) - Expect(err).To(Succeed()) + expectedRouting[endp.EndpointId].Items = append(expectedRouting[endp.EndpointId].Items, cond.GetId()) + } + cond.AlertCondition.AttachedEndpoints = &alertingv1.AttachedEndpoints{ + Items: endps, + InitialDelay: durationpb.New(time.Second * 1), + ThrottlingDuration: durationpb.New(time.Second * 1), + Details: &alertingv1.EndpointImplementation{ + Title: "disconnected agent", + Body: "agent %s is disconnected", + }, } - } - for _, refs := range expectedRouting { - slices.SortFunc(refs.Items, func(a, b *alertingv1.ConditionReference) bool { - if a.GroupId != b.GroupId { - return a.GroupId < b.GroupId - } - return a.Id < b.Id + _, err = alertConditionsClient.UpdateAlertCondition(env.Context(), &alertingv1.UpdateAlertConditionRequest{ + Id: cond.GetId(), + UpdateAlert: cond.AlertCondition, }) - } - - By("creating some default webhook servers as endpoints") - notificationServers = alerting.CreateWebhookServer(env, numNotificationServers) - for _, server := range notificationServers { - ref, err := alertEndpointsClient.CreateAlertEndpoint(env.Context(), server.Endpoint()) Expect(err).To(Succeed()) - server.EndpointId = ref.Id } - endpList, err = alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) - Expect(err).To(Succeed()) - Expect(endpList.Items).To(HaveLen(numNotificationServers + numServers)) + } + for _, refs := range expectedRouting { + slices.SortFunc(refs.Items, func(a, b *alertingv1.ConditionReference) bool { + if a.GroupId != b.GroupId { + return a.GroupId < b.GroupId + } + return a.Id < b.Id + }) + } - By("setting the default servers as default endpoints") - for _, server := range notificationServers { - _, err = alertEndpointsClient.ToggleNotifications(env.Context(), &alertingv1.ToggleRequest{ - Id: &corev1.Reference{Id: server.EndpointId}, - }) - Expect(err).To(Succeed()) - } + By("creating some default webhook servers as endpoints") + notificationServers = alerting.CreateWebhookServer(env, numNotificationServers) + for _, server := range notificationServers { + ref, err := alertEndpointsClient.CreateAlertEndpoint(env.Context(), server.Endpoint()) + Expect(err).To(Succeed()) + server.EndpointId = ref.Id + } + endpList, err = alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) + Expect(err).To(Succeed()) + Expect(endpList.Items).To(HaveLen(numNotificationServers + numServers)) + + By("setting the default servers as default endpoints") + for _, server := range notificationServers { + _, err = alertEndpointsClient.ToggleNotifications(env.Context(), &alertingv1.ToggleRequest{ + Id: &corev1.Reference{Id: server.EndpointId}, + }) + Expect(err).To(Succeed()) + } - By("expecting the conditions to eventually move to the 'OK' state") - Eventually(func() error { - for _, cond := range condList.Items { - status, err := alertConditionsClient.AlertConditionStatus(env.Context(), cond.Id) - if err != nil { - return err - } - if status.State != alertingv1.AlertConditionState_Ok { - return fmt.Errorf("condition %s \"%s\" is not OK, instead in state %s, %s", cond.AlertCondition.Id, cond.AlertCondition.Name, status.State.String(), status.Reason) - } - } - return nil - }, time.Second*30, time.Second).Should(Succeed()) - By("verifying the routing relationships are correctly loaded") - Eventually(func() int { - relationships, err := alertNotificationsClient.ListRoutingRelationships(env.Context(), &emptypb.Empty{}) + By("expecting the conditions to eventually move to the 'OK' state") + Eventually(func() error { + for _, cond := range condList.Items { + status, err := alertConditionsClient.AlertConditionStatus(env.Context(), cond.Id) if err != nil { - return -1 + return err + } + if status.State != alertingv1.AlertConditionState_Ok { + return fmt.Errorf("condition %s \"%s\" is not OK, instead in state %s, %s", cond.AlertCondition.Id, cond.AlertCondition.Name, status.State.String(), status.Reason) } - return len(relationships.RoutingRelationships) - }).Should(Equal(len(expectedRouting))) + } + return nil + }, time.Second*30, time.Second).Should(Succeed()) + By("verifying the routing relationships are correctly loaded") + Eventually(func() int { relationships, err := alertNotificationsClient.ListRoutingRelationships(env.Context(), &emptypb.Empty{}) - Expect(err).To(Succeed()) + if err != nil { + return -1 + } + return len(relationships.RoutingRelationships) + }).Should(Equal(len(expectedRouting))) + relationships, err := alertNotificationsClient.ListRoutingRelationships(env.Context(), &emptypb.Empty{}) + Expect(err).To(Succeed()) - for endpId, rel := range relationships.RoutingRelationships { - slices.SortFunc(rel.Items, func(a, b *alertingv1.ConditionReference) bool { - if a.GroupId != b.GroupId { - return a.GroupId < b.GroupId - } - return a.Id < b.Id - }) - if _, ok := expectedRouting[endpId]; !ok { - Fail(fmt.Sprintf("Expected a routing relation to exist for endpoint id %s", endpId)) - } - Expect(len(rel.Items)).To(Equal(len(expectedRouting[endpId].Items))) - for i := range rel.Items { - Expect(rel.Items[i].Id).To(Equal(expectedRouting[endpId].Items[i].Id)) - Expect(rel.Items[i].GroupId).To(Equal(expectedRouting[endpId].Items[i].GroupId)) + for endpId, rel := range relationships.RoutingRelationships { + slices.SortFunc(rel.Items, func(a, b *alertingv1.ConditionReference) bool { + if a.GroupId != b.GroupId { + return a.GroupId < b.GroupId } + return a.Id < b.Id + }) + if _, ok := expectedRouting[endpId]; !ok { + Fail(fmt.Sprintf("Expected a routing relation to exist for endpoint id %s", endpId)) } - }) + Expect(len(rel.Items)).To(Equal(len(expectedRouting[endpId].Items))) + for i := range rel.Items { + Expect(rel.Items[i].Id).To(Equal(expectedRouting[endpId].Items[i].Id)) + Expect(rel.Items[i].GroupId).To(Equal(expectedRouting[endpId].Items[i].GroupId)) + } + } + }) - Specify("agent disconnect alarms should fire when agents are disconnected ", func() { - // Disconnect a random 3 agents, and verify the servers have the messages - By("disconnecting a random 3 agents") - disconnectedIds := []string{} - toDisconnect := lo.Samples(agents, 3) - for _, disc := range toDisconnect { - disc.CancelFunc() - disconnectedIds = append(disconnectedIds, disc.id) + Specify("agent disconnect alarms should fire when agents are disconnected ", func() { + // Disconnect a random 3 agents, and verify the servers have the messages + By("disconnecting a random 3 agents") + disconnectedIds := []string{} + toDisconnect := lo.Samples(agents, 3) + for _, disc := range toDisconnect { + disc.CancelFunc() + disconnectedIds = append(disconnectedIds, disc.id) + } + + condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) + Expect(err).To(Succeed()) + notInvolvedDisconnects := map[string]struct{}{} + for _, cond := range condList.Items { + if cond.GetAlertCondition().GetAlertType().GetSystem() != nil { + if slices.Contains(disconnectedIds, cond.GetAlertCondition().GetAlertType().GetSystem().ClusterId.Id) { + endps := lo.Map(cond.AlertCondition.AttachedEndpoints.Items, + func(a *alertingv1.AttachedEndpoint, _ int) string { + return a.EndpointId + }) + involvedDisconnects[cond.GetAlertCondition().Id] = endps + } else { + notInvolvedDisconnects[cond.GetAlertCondition().Id] = struct{}{} + } } + } + webhooks := lo.Uniq(lo.Flatten(lo.Values(involvedDisconnects))) + Expect(len(webhooks)).To(BeNumerically(">", 0)) - condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) - Expect(err).To(Succeed()) - notInvolvedDisconnects := map[string]struct{}{} - for _, cond := range condList.Items { - if cond.GetAlertCondition().GetAlertType().GetSystem() != nil { - if slices.Contains(disconnectedIds, cond.GetAlertCondition().GetAlertType().GetSystem().ClusterId.Id) { - endps := lo.Map(cond.AlertCondition.AttachedEndpoints.Items, - func(a *alertingv1.AttachedEndpoint, _ int) string { - return a.EndpointId - }) - involvedDisconnects[cond.GetAlertCondition().Id] = endps - } else { - notInvolvedDisconnects[cond.GetAlertCondition().Id] = struct{}{} + By("verifying the agents are actually disconnected") + Eventually(func() error { + clusters, err := mgmtClient.ListClusters(env.Context(), &managementv1.ListClustersRequest{}) + if err != nil { + return err + } + for _, cl := range clusters.Items { + if slices.Contains(disconnectedIds, cl.GetId()) { + healthStatus, err := mgmtClient.GetClusterHealthStatus(env.Context(), cl.Reference()) + if err != nil { + return err + } + if !healthStatus.Status.Connected == false { + return fmt.Errorf("expected disconnected health status for cluster %s: %s", cl.GetId(), healthStatus.Status.String()) } } } - webhooks := lo.Uniq(lo.Flatten(lo.Values(involvedDisconnects))) - Expect(len(webhooks)).To(BeNumerically(">", 0)) - - By("verifying the agents are actually disconnected") - Eventually(func() error { - clusters, err := mgmtClient.ListClusters(env.Context(), &managementv1.ListClustersRequest{}) + return nil + }, 5*time.Second, 200*time.Millisecond).Should(Succeed()) + + By("verifying the physical servers have received the disconnect messages") + Eventually(func() error { + servers := servers + conditionIds := lo.Keys(involvedDisconnects) + for _, id := range conditionIds { + status, err := alertConditionsClient.AlertConditionStatus(env.Context(), &alertingv1.ConditionReference{Id: id}) if err != nil { return err } - for _, cl := range clusters.Items { - if slices.Contains(disconnectedIds, cl.GetId()) { - healthStatus, err := mgmtClient.GetClusterHealthStatus(env.Context(), cl.Reference()) - if err != nil { - return err - } - if !healthStatus.Status.Connected == false { - return fmt.Errorf("expected disconnected health status for cluster %s: %s", cl.GetId(), healthStatus.Status.String()) - } - } + if status.GetState() != alertingv1.AlertConditionState_Firing { + return fmt.Errorf("expected alerting condition %s to be firing, got %s", id, status.GetState().String()) } - return nil - }, 5*time.Second, 200*time.Millisecond).Should(Succeed()) + } - By("verifying the physical servers have received the disconnect messages") - Eventually(func() error { - servers := servers - conditionIds := lo.Keys(involvedDisconnects) - for _, id := range conditionIds { - status, err := alertConditionsClient.AlertConditionStatus(env.Context(), &alertingv1.ConditionReference{Id: id}) - if err != nil { - return err - } - if status.GetState() != alertingv1.AlertConditionState_Firing { - return fmt.Errorf("expected alerting condition %s to be firing, got %s", id, status.GetState().String()) - } + for id := range notInvolvedDisconnects { + status, err := alertConditionsClient.AlertConditionStatus(env.Context(), &alertingv1.ConditionReference{Id: id}) + if err != nil { + return err } - - for id := range notInvolvedDisconnects { - status, err := alertConditionsClient.AlertConditionStatus(env.Context(), &alertingv1.ConditionReference{Id: id}) - if err != nil { - return err - } - if status.GetState() != alertingv1.AlertConditionState_Ok { - return fmt.Errorf("expected unaffected alerting condition %s to be ok, got %s", id, status.GetState().String()) - } + if status.GetState() != alertingv1.AlertConditionState_Ok { + return fmt.Errorf("expected unaffected alerting condition %s to be ok, got %s", id, status.GetState().String()) } + } - for _, server := range servers { - if slices.Contains(webhooks, server.EndpointId) { - // hard to map these excatly without recreating the internal routing logic from the routers - // since we have dedicated routing integration tests, we can just check that the buffer is not empty - if len(server.GetBuffer()) == 0 { - return fmt.Errorf("expected webhook server %s to have messages, got %d", server.EndpointId, len(server.GetBuffer())) - } + for _, server := range servers { + if slices.Contains(webhooks, server.EndpointId) { + // hard to map these excatly without recreating the internal routing logic from the routers + // since we have dedicated routing integration tests, we can just check that the buffer is not empty + if len(server.GetBuffer()) == 0 { + return fmt.Errorf("expected webhook server %s to have messages, got %d", server.EndpointId, len(server.GetBuffer())) } } - return nil - }, time.Second*60, time.Millisecond*500).Should(Succeed()) + } + return nil + }, time.Second*60, time.Millisecond*500).Should(Succeed()) - By("verifying the notification servers have not received any alarm disconnect messages") - Eventually(func() error { - for _, server := range notificationServers { - if len(server.GetBuffer()) != 0 { - return fmt.Errorf("expected webhook server %s to not have any notifications, got %d", server.EndpointId, len(server.GetBuffer())) - } + By("verifying the notification servers have not received any alarm disconnect messages") + Eventually(func() error { + for _, server := range notificationServers { + if len(server.GetBuffer()) != 0 { + return fmt.Errorf("expected webhook server %s to not have any notifications, got %d", server.EndpointId, len(server.GetBuffer())) } - return nil - }, time.Second*5, time.Second*1).Should(Succeed()) - }) + } + return nil + }, time.Second*5, time.Second*1).Should(Succeed()) + }) - It("should be able to batch list status and filter by status", func() { - condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) - Expect(err).To(Succeed()) + It("should be able to batch list status and filter by status", func() { + condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) + Expect(err).To(Succeed()) - statusCondList, err := alertConditionsClient.ListAlertConditionsWithStatus(env.Context(), &alertingv1.ListStatusRequest{}) - Expect(err).To(Succeed()) - Expect(statusCondList.AlertConditions).To(HaveLen(len(condList.Items))) - for condId, cond := range statusCondList.AlertConditions { - if slices.Contains(lo.Keys(involvedDisconnects), condId) { - Expect(cond.Status.State).To(Equal(alertingv1.AlertConditionState_Firing)) - } else { - Expect(cond.Status.State).To(Equal(alertingv1.AlertConditionState_Ok)) - } - } - firingOnlyStatusList, err := alertConditionsClient.ListAlertConditionsWithStatus(env.Context(), &alertingv1.ListStatusRequest{ - States: []alertingv1.AlertConditionState{ - alertingv1.AlertConditionState_Firing, - }, - }) - Expect(err).To(Succeed()) - Expect(firingOnlyStatusList.AlertConditions).To(HaveLen(len(involvedDisconnects))) - for _, cond := range firingOnlyStatusList.AlertConditions { + statusCondList, err := alertConditionsClient.ListAlertConditionsWithStatus(env.Context(), &alertingv1.ListStatusRequest{}) + Expect(err).To(Succeed()) + Expect(statusCondList.AlertConditions).To(HaveLen(len(condList.Items))) + for condId, cond := range statusCondList.AlertConditions { + if slices.Contains(lo.Keys(involvedDisconnects), condId) { Expect(cond.Status.State).To(Equal(alertingv1.AlertConditionState_Firing)) + } else { + Expect(cond.Status.State).To(Equal(alertingv1.AlertConditionState_Ok)) } + } + firingOnlyStatusList, err := alertConditionsClient.ListAlertConditionsWithStatus(env.Context(), &alertingv1.ListStatusRequest{ + States: []alertingv1.AlertConditionState{ + alertingv1.AlertConditionState_Firing, + }, }) + Expect(err).To(Succeed()) + Expect(firingOnlyStatusList.AlertConditions).To(HaveLen(len(involvedDisconnects))) + for _, cond := range firingOnlyStatusList.AlertConditions { + Expect(cond.Status.State).To(Equal(alertingv1.AlertConditionState_Firing)) + } + }) - It("should be able to push notifications to our notification endpoints", func() { - Expect(len(notificationServers)).To(BeNumerically(">", 0)) - By("forwarding the message to AlertManager") - _, err := alertNotificationsClient.PushNotification(env.Context(), &alertingv1.Notification{ - Title: "hello", - Body: "world", - // set to critical in order to expedite the notification during testing - Properties: map[string]string{ - message.NotificationPropertySeverity: alertingv1.OpniSeverity_Critical.String(), - }, - }) - Expect(err).To(Succeed()) - - By("verifying the endpoints have received the notification messages") - Eventually(func() error { - for _, server := range notificationServers { - if len(server.GetBuffer()) == 0 { - return fmt.Errorf("expected webhook server %s to have messages, got %d", server.EndpointId, len(server.GetBuffer())) - } - } - return nil - }, time.Second*60, time.Second).Should(Succeed()) + It("should be able to push notifications to our notification endpoints", func() { + Expect(len(notificationServers)).To(BeNumerically(">", 0)) + By("forwarding the message to AlertManager") + _, err := alertNotificationsClient.PushNotification(env.Context(), &alertingv1.Notification{ + Title: "hello", + Body: "world", + // set to critical in order to expedite the notification during testing + Properties: map[string]string{ + message.NotificationPropertySeverity: alertingv1.OpniSeverity_Critical.String(), + }, }) + Expect(err).To(Succeed()) - It("should be able to list opni messages", func() { - Eventually(func() error { - list, err := alertNotificationsClient.ListNotifications(env.Context(), &alertingv1.ListNotificationRequest{}) - if err != nil { - return err - } - if len(list.Items) == 0 { - return fmt.Errorf("expected to find at least one notification, got 0") + By("verifying the endpoints have received the notification messages") + Eventually(func() error { + for _, server := range notificationServers { + if len(server.GetBuffer()) == 0 { + return fmt.Errorf("expected webhook server %s to have messages, got %d", server.EndpointId, len(server.GetBuffer())) } - return nil - }, time.Second*60, time.Second).Should(Succeed()) + } + return nil + }, time.Second*60, time.Second).Should(Succeed()) + }) - By("verifying we enforce limits") - list, err := alertNotificationsClient.ListNotifications(env.Context(), &alertingv1.ListNotificationRequest{ - Limit: lo.ToPtr(int32(1)), - }) - Expect(err).To(Succeed()) - Expect(len(list.Items)).To(Equal(1)) + It("should be able to list opni messages", func() { + Eventually(func() error { + list, err := alertNotificationsClient.ListNotifications(env.Context(), &alertingv1.ListNotificationRequest{}) + if err != nil { + return err + } + if len(list.Items) == 0 { + return fmt.Errorf("expected to find at least one notification, got 0") + } + return nil + }, time.Second*60, time.Second).Should(Succeed()) + + By("verifying we enforce limits") + list, err := alertNotificationsClient.ListNotifications(env.Context(), &alertingv1.ListNotificationRequest{ + Limit: lo.ToPtr(int32(1)), }) + Expect(err).To(Succeed()) + Expect(len(list.Items)).To(Equal(1)) + }) - It("should return warnings when trying to edit/delete alert endpoints that are involved in conditions", func() { - webhooks := lo.Uniq(lo.Flatten(lo.Values(involvedDisconnects))) - Expect(len(webhooks)).To(BeNumerically(">", 0)) + It("should return warnings when trying to edit/delete alert endpoints that are involved in conditions", func() { + webhooks := lo.Uniq(lo.Flatten(lo.Values(involvedDisconnects))) + Expect(len(webhooks)).To(BeNumerically(">", 0)) - for _, webhook := range webhooks { - involvedConditions, err := alertEndpointsClient.UpdateAlertEndpoint(env.Context(), &alertingv1.UpdateAlertEndpointRequest{ - Id: &corev1.Reference{ - Id: webhook, - }, - UpdateAlert: &alertingv1.AlertEndpoint{ - Name: "update", - Description: "update", - Endpoint: &alertingv1.AlertEndpoint_Webhook{ - Webhook: &alertingv1.WebhookEndpoint{ - Url: "http://example.com", - }, + for _, webhook := range webhooks { + involvedConditions, err := alertEndpointsClient.UpdateAlertEndpoint(env.Context(), &alertingv1.UpdateAlertEndpointRequest{ + Id: &corev1.Reference{ + Id: webhook, + }, + UpdateAlert: &alertingv1.AlertEndpoint{ + Name: "update", + Description: "update", + Endpoint: &alertingv1.AlertEndpoint_Webhook{ + Webhook: &alertingv1.WebhookEndpoint{ + Url: "http://example.com", }, - Id: "id", }, - ForceUpdate: false, - }) - Expect(err).NotTo(HaveOccurred()) + Id: "id", + }, + ForceUpdate: false, + }) + Expect(err).NotTo(HaveOccurred()) - Expect(involvedConditions.Items).NotTo(HaveLen(0)) - involvedConditions, err = alertEndpointsClient.DeleteAlertEndpoint(env.Context(), &alertingv1.DeleteAlertEndpointRequest{ - Id: &corev1.Reference{ - Id: webhook, - }, - ForceDelete: false, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(involvedConditions.Items).NotTo(HaveLen(0)) - } - }) + Expect(involvedConditions.Items).NotTo(HaveLen(0)) + involvedConditions, err = alertEndpointsClient.DeleteAlertEndpoint(env.Context(), &alertingv1.DeleteAlertEndpointRequest{ + Id: &corev1.Reference{ + Id: webhook, + }, + ForceDelete: false, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(involvedConditions.Items).NotTo(HaveLen(0)) + } + }) - It("should have a functional timeline", func() { - condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) - Expect(err).To(Succeed()) + It("should have a functional timeline", func() { + condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) + Expect(err).To(Succeed()) - // By("verifying the timeline shows only the firing conditions ") - Eventually(func() error { - timeline, err := alertConditionsClient.Timeline(env.Context(), &alertingv1.TimelineRequest{ - LookbackWindow: durationpb.New(time.Minute * 5), - }) - if err != nil { - return err - } + // By("verifying the timeline shows only the firing conditions ") + Eventually(func() error { + timeline, err := alertConditionsClient.Timeline(env.Context(), &alertingv1.TimelineRequest{ + LookbackWindow: durationpb.New(time.Minute * 5), + }) + if err != nil { + return err + } - By("verifying the timeline matches the conditions") - if len(timeline.Items) != len(condList.Items) { - return fmt.Errorf("expected timeline to have %d items, got %d", len(condList.Items), len(timeline.Items)) - } + By("verifying the timeline matches the conditions") + if len(timeline.Items) != len(condList.Items) { + return fmt.Errorf("expected timeline to have %d items, got %d", len(condList.Items), len(timeline.Items)) + } - for id, item := range timeline.GetItems() { - if slices.Contains(lo.Keys(involvedDisconnects), id) { - if len(item.Windows) == 0 { - return fmt.Errorf("firing condition should show up on timeline, but does not") - } - if len(item.Windows) != 1 { - return fmt.Errorf("condition evaluation is flaky, should only have one window, but has %d", len(item.Windows)) - } - messages, err := alertNotificationsClient.ListAlarmMessages(env.Context(), &alertingv1.ListAlarmMessageRequest{ - ConditionId: &alertingv1.ConditionReference{ - Id: id, - }, - Fingerprints: item.Windows[0].Fingerprints, - Start: item.Windows[0].Start, - End: timestamppb.Now(), - }) - if err != nil { - return err - } - if !(len(messages.Items) > 0) { - return fmt.Errorf("expected firing condition to have cached messages") - } - Expect(len(messages.Items)).To(BeNumerically(">", 0)) + for id, item := range timeline.GetItems() { + if slices.Contains(lo.Keys(involvedDisconnects), id) { + if len(item.Windows) == 0 { + return fmt.Errorf("firing condition should show up on timeline, but does not") + } + if len(item.Windows) != 1 { + return fmt.Errorf("condition evaluation is flaky, should only have one window, but has %d", len(item.Windows)) + } + messages, err := alertNotificationsClient.ListAlarmMessages(env.Context(), &alertingv1.ListAlarmMessageRequest{ + ConditionId: &alertingv1.ConditionReference{ + Id: id, + }, + Fingerprints: item.Windows[0].Fingerprints, + Start: item.Windows[0].Start, + End: timestamppb.Now(), + }) + if err != nil { + return err + } + if !(len(messages.Items) > 0) { + return fmt.Errorf("expected firing condition to have cached messages") + } + Expect(len(messages.Items)).To(BeNumerically(">", 0)) - } else { - if len(item.Windows) != 0 { - return fmt.Errorf("conditions that have not fired should not show up on timeline, but do") - } + } else { + if len(item.Windows) != 0 { + return fmt.Errorf("conditions that have not fired should not show up on timeline, but do") } } - return nil - }, time.Second*15, time.Second).Should(Succeed()) - }) + } + return nil + }, time.Second*15, time.Second).Should(Succeed()) + }) - Specify("the alertmanager proxy served by the Gateway HTTP port should be able to list the alarms", func() { - alertingProxyGET := fmt.Sprintf("https://%s/plugin_alerting/alertmanager/api/v2/alerts/groups", env.GatewayConfig().Spec.HTTPListenAddress) - req, err := http.NewRequestWithContext(env.Context(), http.MethodGet, alertingProxyGET, nil) - Expect(err).To(Succeed()) + Specify("the alertmanager proxy served by the Gateway HTTP port should be able to list the alarms", func() { + alertingProxyGET := fmt.Sprintf("https://%s/plugin_alerting/alertmanager/api/v2/alerts/groups", env.GatewayConfig().Spec.HTTPListenAddress) + req, err := http.NewRequestWithContext(env.Context(), http.MethodGet, alertingProxyGET, nil) + Expect(err).To(Succeed()) - Eventually(func() error { - resp, err := httpProxyClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("expected proxy to return status OK, instead got : %d, %s", resp.StatusCode, resp.Status) - } - res := alertmanagerv2.AlertGroups{} - if err := json.NewDecoder(resp.Body).Decode(&res); err != nil { - return err - } - if len(res) == 0 { - return fmt.Errorf("expected to get non-empty alertgroup from alertmanager") - } - return nil - }).Should(Succeed()) - }) + Eventually(func() error { + resp, err := httpProxyClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("expected proxy to return status OK, instead got : %d, %s", resp.StatusCode, resp.Status) + } + res := alertmanagerv2.AlertGroups{} + if err := json.NewDecoder(resp.Body).Decode(&res); err != nil { + return err + } + if len(res) == 0 { + return fmt.Errorf("expected to get non-empty alertgroup from alertmanager") + } + return nil + }).Should(Succeed()) + }) - It("should sync friendly cluster names to conditions", func() { - friendlyName := "friendly-name" - agentId := agents[0].id - cl, err := mgmtClient.GetCluster(env.Context(), &corev1.Reference{Id: agentId}) - Expect(err).To(Succeed()) + It("should sync friendly cluster names to conditions", func() { + friendlyName := "friendly-name" + agentId := agents[0].id + cl, err := mgmtClient.GetCluster(env.Context(), &corev1.Reference{Id: agentId}) + Expect(err).To(Succeed()) + + _, err = mgmtClient.EditCluster(env.Context(), &managementv1.EditClusterRequest{ + Cluster: &corev1.Reference{Id: agentId}, + Labels: lo.Assign(cl.GetLabels(), map[string]string{ + corev1.NameLabel: friendlyName, + }), + }) + Expect(err).To(Succeed()) - _, err = mgmtClient.EditCluster(env.Context(), &managementv1.EditClusterRequest{ - Cluster: &corev1.Reference{Id: agentId}, - Labels: lo.Assign(cl.GetLabels(), map[string]string{ - corev1.NameLabel: friendlyName, - }), + Eventually(func() error { + conds, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ + Clusters: []string{agentId}, }) - Expect(err).To(Succeed()) + if err != nil { + return err + } - Eventually(func() error { - conds, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ - Clusters: []string{agentId}, - }) - if err != nil { - return err + for _, cond := range conds.Items { + name, ok := cond.GetAlertCondition().Annotations[message.NotificationContentClusterName] + if !ok { + return fmt.Errorf("expected alert condition to contain annotations with cluster name") } - - for _, cond := range conds.Items { - name, ok := cond.GetAlertCondition().Annotations[message.NotificationContentClusterName] - if !ok { - return fmt.Errorf("expected alert condition to contain annotations with cluster name") - } - if name != friendlyName { - return fmt.Errorf("expected alert condition to contain annotations with cluster name %s, got %s", friendlyName, name) - } + if name != friendlyName { + return fmt.Errorf("expected alert condition to contain annotations with cluster name %s, got %s", friendlyName, name) } - return nil - }, time.Second*5, time.Millisecond*300).Should(Succeed()) + } + return nil + }, time.Second*5, time.Millisecond*300).Should(Succeed()) - }) + }) - It("should force update/delete alert endpoints involved in conditions", func() { - By("verifying we can edit Alert Endpoints in use by Alert Conditions") - endpList, err := alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) - Expect(err).NotTo(HaveOccurred()) - Expect(len(endpList.Items)).To(BeNumerically(">", 0)) - Expect(endpList.Items).To(HaveLen(numServers + numNotificationServers)) - for _, endp := range endpList.Items { - _, err := alertEndpointsClient.UpdateAlertEndpoint(env.Context(), &alertingv1.UpdateAlertEndpointRequest{ - Id: &corev1.Reference{ - Id: endp.Id.Id, - }, - UpdateAlert: &alertingv1.AlertEndpoint{ - Name: "update", - Description: "update", - Endpoint: &alertingv1.AlertEndpoint_Webhook{ - Webhook: &alertingv1.WebhookEndpoint{ - Url: "http://example.com", - }, + It("should force update/delete alert endpoints involved in conditions", func() { + By("verifying we can edit Alert Endpoints in use by Alert Conditions") + endpList, err := alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(len(endpList.Items)).To(BeNumerically(">", 0)) + Expect(endpList.Items).To(HaveLen(numServers + numNotificationServers)) + for _, endp := range endpList.Items { + _, err := alertEndpointsClient.UpdateAlertEndpoint(env.Context(), &alertingv1.UpdateAlertEndpointRequest{ + Id: &corev1.Reference{ + Id: endp.Id.Id, + }, + UpdateAlert: &alertingv1.AlertEndpoint{ + Name: "update", + Description: "update", + Endpoint: &alertingv1.AlertEndpoint_Webhook{ + Webhook: &alertingv1.WebhookEndpoint{ + Url: "http://example.com", }, - Id: endp.Id.Id, }, - ForceUpdate: true, - }) - Expect(err).NotTo(HaveOccurred()) - } - endpList, err = alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) - Expect(err).NotTo(HaveOccurred()) - Expect(len(endpList.Items)).To(BeNumerically(">", 0)) - Expect(endpList.Items).To(HaveLen(numServers + numNotificationServers)) - updatedList := lo.Filter(endpList.Items, func(item *alertingv1.AlertEndpointWithId, _ int) bool { - if item.Endpoint.GetWebhook() != nil { - return item.Endpoint.GetName() == "update" && item.Endpoint.GetDescription() == "update" - } - return false + Id: endp.Id.Id, + }, + ForceUpdate: true, }) - Expect(updatedList).To(HaveLen(len(endpList.Items))) - - By("verifying we can delete Alert Endpoint in use by Alert Conditions") - for _, endp := range endpList.Items { - _, err := alertEndpointsClient.DeleteAlertEndpoint(env.Context(), &alertingv1.DeleteAlertEndpointRequest{ - Id: &corev1.Reference{ - Id: endp.Id.Id, - }, - ForceDelete: true, - }) - Expect(err).NotTo(HaveOccurred()) - } - endpList, err = alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) Expect(err).NotTo(HaveOccurred()) - Expect(endpList.Items).To(HaveLen(0)) - - condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) - Expect(err).NotTo(HaveOccurred()) - Expect(condList.Items).NotTo(HaveLen(0)) - hasEndpoints := lo.Filter(condList.Items, func(item *alertingv1.AlertConditionWithId, _ int) bool { - if item.AlertCondition.AttachedEndpoints != nil { - return len(item.AlertCondition.AttachedEndpoints.Items) != 0 - } - return false - }) - Expect(hasEndpoints).To(HaveLen(0)) + } + endpList, err = alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(len(endpList.Items)).To(BeNumerically(">", 0)) + Expect(endpList.Items).To(HaveLen(numServers + numNotificationServers)) + updatedList := lo.Filter(endpList.Items, func(item *alertingv1.AlertEndpointWithId, _ int) bool { + if item.Endpoint.GetWebhook() != nil { + return item.Endpoint.GetName() == "update" && item.Endpoint.GetDescription() == "update" + } + return false }) + Expect(updatedList).To(HaveLen(len(endpList.Items))) - It("should delete the downstream agents", func() { - client := env.NewManagementClient() - agents, err := client.ListClusters(env.Context(), &managementv1.ListClustersRequest{}) + By("verifying we can delete Alert Endpoint in use by Alert Conditions") + for _, endp := range endpList.Items { + _, err := alertEndpointsClient.DeleteAlertEndpoint(env.Context(), &alertingv1.DeleteAlertEndpointRequest{ + Id: &corev1.Reference{ + Id: endp.Id.Id, + }, + ForceDelete: true, + }) Expect(err).NotTo(HaveOccurred()) - for _, agent := range agents.Items { - _, err := client.DeleteCluster(env.Context(), agent.Reference()) - Expect(err).NotTo(HaveOccurred()) + } + endpList, err = alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(endpList.Items).To(HaveLen(0)) + + condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(condList.Items).NotTo(HaveLen(0)) + hasEndpoints := lo.Filter(condList.Items, func(item *alertingv1.AlertConditionWithId, _ int) bool { + if item.AlertCondition.AttachedEndpoints != nil { + return len(item.AlertCondition.AttachedEndpoints.Items) != 0 } + return false }) + Expect(hasEndpoints).To(HaveLen(0)) + }) - It("should uninstall the alerting cluster", func() { - _, err := alertClusterClient.UninstallCluster(env.Context(), &alertops.UninstallRequest{ - DeleteData: true, - }) - Expect(err).To(BeNil()) + It("should delete the downstream agents", func() { + client := env.NewManagementClient() + agents, err := client.ListClusters(env.Context(), &managementv1.ListClustersRequest{}) + Expect(err).NotTo(HaveOccurred()) + for _, agent := range agents.Items { + _, err := client.DeleteCluster(env.Context(), agent.Reference()) + Expect(err).NotTo(HaveOccurred()) + } + }) + + It("should uninstall the alerting cluster", func() { + _, err := alertClusterClient.UninstallCluster(env.Context(), &alertops.UninstallRequest{ + DeleteData: true, }) - } - }) + Expect(err).To(BeNil()) + }) + } }) -} +}) diff --git a/test/plugins/alerting/routing_test.go b/test/plugins/alerting/routing_test.go index 3768c79842..ad083ea688 100644 --- a/test/plugins/alerting/routing_test.go +++ b/test/plugins/alerting/routing_test.go @@ -18,9 +18,9 @@ import ( "github.com/rancher/opni/pkg/alerting/drivers/routing" "github.com/rancher/opni/pkg/alerting/shared" alertingv1 "github.com/rancher/opni/pkg/apis/alerting/v1" + "github.com/rancher/opni/pkg/test" "github.com/rancher/opni/pkg/test/alerting" "github.com/rancher/opni/pkg/test/freeport" - "github.com/rancher/opni/pkg/test/testruntime" "github.com/rancher/opni/pkg/util" "github.com/samber/lo" "golang.org/x/exp/slices" @@ -29,225 +29,226 @@ import ( var defaultHook *alerting.MockIntegrationWebhookServer -func init() { - testruntime.IfIntegration(func() { - BuildRoutingLogicTest( - func() routing.OpniRouting { - defaultHooks := alerting.NewWebhookMemoryServer(env, "webhook") - defaultHook = defaultHooks - cfg := config.WebhookConfig{ - NotifierConfig: config.NotifierConfig{ - VSendResolved: false, - }, - URL: &amCfg.URL{ - URL: util.Must(url.Parse(defaultHook.GetWebhook())), - }, - } - return routing.NewOpniRouterV1(cfg) - }, - ) +var _ = Describe("Alerting routing logic translation to physical dispatching", Ordered, Label("integration"), func() { + var alertingClient client.AlertingClient + var alertingClient2 client.AlertingClient + var alertingClient3 client.AlertingClient + var router routing.OpniRouting + var env *test.Environment + var tmpConfigDir string + + BeforeAll(func() { + env = &test.Environment{} + Expect(env).NotTo(BeNil()) + Expect(env.Start()).To(Succeed()) + DeferCleanup(env.Stop) + tmpConfigDir = env.GenerateNewTempDirectory("alertmanager-config") + Expect(tmpConfigDir).NotTo(Equal("")) + + routerConstructor := func() routing.OpniRouting { + defaultHooks := alerting.NewWebhookMemoryServer(env, "webhook") + defaultHook = defaultHooks + cfg := config.WebhookConfig{ + NotifierConfig: config.NotifierConfig{ + VSendResolved: false, + }, + URL: &amCfg.URL{ + URL: util.Must(url.Parse(defaultHook.GetWebhook())), + }, + } + return routing.NewOpniRouterV1(cfg) + } + + router = routerConstructor() + Expect(router).NotTo(BeNil()) }) -} -func BuildRoutingLogicTest( - routerConstructor func() routing.OpniRouting, -) bool { - return Describe("Alerting routing logic translation to physical dispatching", Ordered, Label("integration"), func() { - var alertingClient client.AlertingClient - var alertingClient2 client.AlertingClient - var alertingClient3 client.AlertingClient - When("setting namespace specs on the routing tree", func() { - step := "initial" - var router routing.OpniRouting - BeforeAll(func() { - Expect(env).NotTo(BeNil()) - router = routerConstructor() - Expect(router).NotTo(BeNil()) - }) - AfterEach(func() { - By(fmt.Sprintf("%s step: expecting that the router can build the config", step)) - currentCfg, err := router.BuildConfig() - Expect(err).To(Succeed()) - By(fmt.Sprintf("%s step: expecting that the formed alertmanager config is correct", step)) - fp := freeport.GetFreePort() + When("setting namespace specs on the routing tree", func() { + step := "initial" - alerting.ExpectAlertManagerConfigToBeValid(env.Context(), env, tmpConfigDir, step+".yaml", currentCfg, fp) - }) + AfterEach(func() { + By(fmt.Sprintf("%s step: expecting that the router can build the config", step)) + currentCfg, err := router.BuildConfig() + Expect(err).To(Succeed()) + By(fmt.Sprintf("%s step: expecting that the formed alertmanager config is correct", step)) + fp := freeport.GetFreePort() - It("should be able to dynamically update alert routing", func() { - step = "dynamic-alert-routing" - tmpConfigDir := env.GenerateNewTempDirectory("webhook") - err := os.MkdirAll(tmpConfigDir, 0755) - Expect(err).To(Succeed()) - By("Creating some test webhook servers") + alerting.ExpectAlertManagerConfigToBeValid(env.Context(), env, tmpConfigDir, step+".yaml", currentCfg, fp) + }) - servers := alerting.CreateWebhookServer(env, 3) - server1, server2, server3 := servers[0], servers[1], servers[2] + It("should be able to dynamically update alert routing", func() { + step = "dynamic-alert-routing" + tmpConfigDir := env.GenerateNewTempDirectory("webhook") + err := os.MkdirAll(tmpConfigDir, 0755) + Expect(err).To(Succeed()) + By("Creating some test webhook servers") - condId1, condId2, condId3 := uuid.New().String(), uuid.New().String(), uuid.New().String() - ns := "test" - By("routing to a subset of the test webhook servers") - details1 := &alertingv1.EndpointImplementation{ - Title: "test1", - Body: "test1", - } - details2 := &alertingv1.EndpointImplementation{ - Title: "test2", - Body: "test2", - } - details3 := &alertingv1.EndpointImplementation{ - Title: "test3", - Body: "test3", - } - suiteSpec := &testSpecSuite{ - name: "dynamic-alert-routing", - defaultServer: defaultHook, - specs: []*testSpec{ - { - namespace: ns, - id: condId1, - servers: []*alerting.MockIntegrationWebhookServer{server1}, - details: details1, - }, - { - namespace: ns, - id: condId2, - servers: []*alerting.MockIntegrationWebhookServer{server1, server2}, - details: details2, - }, - { - namespace: ns, - id: condId3, - servers: []*alerting.MockIntegrationWebhookServer{server1, server2, server3}, - details: details3, - }, - }, - } - By("setting the router to the namespace specs") - for _, spec := range suiteSpec.specs { - endpoints := lo.Map( - spec.servers, - func(server *alerting.MockIntegrationWebhookServer, _ int) *alertingv1.FullAttachedEndpoint { - return &alertingv1.FullAttachedEndpoint{ - AlertEndpoint: server.Endpoint(), - EndpointId: server.Endpoint().Id, - Details: spec.details, - } - }) - err = router.SetNamespaceSpec( - spec.namespace, - spec.id, - &alertingv1.FullAttachedEndpoints{ - Items: endpoints, - Details: spec.details, - InitialDelay: durationpb.New(time.Second * 1), - ThrottlingDuration: durationpb.New(time.Second * 1), - }, - ) - Expect(err).To(Succeed()) - } + servers := alerting.CreateWebhookServer(env, 3) + server1, server2, server3 := servers[0], servers[1], servers[2] - By("running alertmanager with this config") - amPort, ca := alerting.RunAlertManager(env, router, tmpConfigDir, step+".yaml") - alertingClient = client.NewClient( - nil, - fmt.Sprintf("http://localhost:%d", amPort), - fmt.Sprintf("http://localhost:%d", 0), - ) - defer ca() - By("sending alerts to each condition in the router") - for _, spec := range suiteSpec.specs { - err := alertingClient.AlertClient().PostAlarm(context.TODO(), client.AlertObject{ - Id: spec.id, - Labels: map[string]string{ - ns: spec.id, - }, - Annotations: map[string]string{}, + condId1, condId2, condId3 := uuid.New().String(), uuid.New().String(), uuid.New().String() + ns := "test" + By("routing to a subset of the test webhook servers") + details1 := &alertingv1.EndpointImplementation{ + Title: "test1", + Body: "test1", + } + details2 := &alertingv1.EndpointImplementation{ + Title: "test2", + Body: "test2", + } + details3 := &alertingv1.EndpointImplementation{ + Title: "test3", + Body: "test3", + } + suiteSpec := &testSpecSuite{ + name: "dynamic-alert-routing", + defaultServer: defaultHook, + specs: []*testSpec{ + { + namespace: ns, + id: condId1, + servers: []*alerting.MockIntegrationWebhookServer{server1}, + details: details1, + }, + { + namespace: ns, + id: condId2, + servers: []*alerting.MockIntegrationWebhookServer{server1, server2}, + details: details2, + }, + { + namespace: ns, + id: condId3, + servers: []*alerting.MockIntegrationWebhookServer{server1, server2, server3}, + details: details3, + }, + }, + } + By("setting the router to the namespace specs") + for _, spec := range suiteSpec.specs { + endpoints := lo.Map( + spec.servers, + func(server *alerting.MockIntegrationWebhookServer, _ int) *alertingv1.FullAttachedEndpoint { + return &alertingv1.FullAttachedEndpoint{ + AlertEndpoint: server.Endpoint(), + EndpointId: server.Endpoint().Id, + Details: spec.details, + } }) - Expect(err).To(Succeed()) - } - Eventually(func() error { - return suiteSpec.ExpectAlertsToBeRouted(amPort) - }, time.Second*30, time.Second*1).Should(Succeed()) - ca() - server1.ClearBuffer() - server2.ClearBuffer() - server3.ClearBuffer() - defaultHook.ClearBuffer() + err = router.SetNamespaceSpec( + spec.namespace, + spec.id, + &alertingv1.FullAttachedEndpoints{ + Items: endpoints, + Details: spec.details, + InitialDelay: durationpb.New(time.Second * 1), + ThrottlingDuration: durationpb.New(time.Second * 1), + }, + ) + Expect(err).To(Succeed()) + } - By("deleting a random server endpoint") - // ok - err = router.DeleteEndpoint(suiteSpec.specs[0].servers[0].Endpoint().Id) + By("running alertmanager with this config") + amPort, ca := alerting.RunAlertManager(env, router, tmpConfigDir, step+".yaml") + alertingClient = client.NewClient( + nil, + fmt.Sprintf("http://localhost:%d", amPort), + fmt.Sprintf("http://localhost:%d", 0), + ) + defer ca() + By("sending alerts to each condition in the router") + for _, spec := range suiteSpec.specs { + err := alertingClient.AlertClient().PostAlarm(context.TODO(), client.AlertObject{ + Id: spec.id, + Labels: map[string]string{ + ns: spec.id, + }, + Annotations: map[string]string{}, + }) Expect(err).To(Succeed()) - for _, spec := range suiteSpec.specs { - spec.servers = spec.servers[1:] - } + } + Eventually(func() error { + return suiteSpec.ExpectAlertsToBeRouted(amPort) + }, time.Second*30, time.Second*1).Should(Succeed()) + ca() + server1.ClearBuffer() + server2.ClearBuffer() + server3.ClearBuffer() + defaultHook.ClearBuffer() - amPort2, ca2 := alerting.RunAlertManager(env, router, tmpConfigDir, step+".yaml") - alertingClient2 = client.NewClient( - nil, - fmt.Sprintf("http://localhost:%d", amPort2), - fmt.Sprintf("http://localhost:%d", 0), - ) - defer ca2() - By("sending alerts to each condition in the router") - for _, spec := range suiteSpec.specs { - err := alertingClient2.AlertClient().PostAlarm(context.TODO(), client.AlertObject{ - Id: spec.id, - Labels: map[string]string{ - ns: spec.id, - }, - Annotations: map[string]string{}, - }) - Expect(err).To(Succeed()) - } - Eventually(func() error { - return suiteSpec.ExpectAlertsToBeRouted(amPort2) - }, time.Second*30, time.Second*1).Should(Succeed()) - ca2() + By("deleting a random server endpoint") + // ok + err = router.DeleteEndpoint(suiteSpec.specs[0].servers[0].Endpoint().Id) + Expect(err).To(Succeed()) + for _, spec := range suiteSpec.specs { + spec.servers = spec.servers[1:] + } - By("updating an endpoint to another endpoint") + amPort2, ca2 := alerting.RunAlertManager(env, router, tmpConfigDir, step+".yaml") + alertingClient2 = client.NewClient( + nil, + fmt.Sprintf("http://localhost:%d", amPort2), + fmt.Sprintf("http://localhost:%d", 0), + ) + defer ca2() + By("sending alerts to each condition in the router") + for _, spec := range suiteSpec.specs { + err := alertingClient2.AlertClient().PostAlarm(context.TODO(), client.AlertObject{ + Id: spec.id, + Labels: map[string]string{ + ns: spec.id, + }, + Annotations: map[string]string{}, + }) + Expect(err).To(Succeed()) + } + Eventually(func() error { + return suiteSpec.ExpectAlertsToBeRouted(amPort2) + }, time.Second*30, time.Second*1).Should(Succeed()) + ca2() - server1.ClearBuffer() - server2.ClearBuffer() - server3.ClearBuffer() - defaultHook.ClearBuffer() + By("updating an endpoint to another endpoint") - err = router.UpdateEndpoint(server2.Endpoint().Id, server1.Endpoint()) - Expect(err).To(Succeed()) - for _, spec := range suiteSpec.specs { - if len(spec.servers) != 0 { - spec.servers[0] = server1 - } - } + server1.ClearBuffer() + server2.ClearBuffer() + server3.ClearBuffer() + defaultHook.ClearBuffer() - By("send an an alert to each specs") - amPort3, ca3 := alerting.RunAlertManager(env, router, tmpConfigDir, step+".yaml") - defer ca3() - alertingClient3 = client.NewClient( - nil, - fmt.Sprintf("http://localhost:%d", amPort3), - fmt.Sprintf("http://localhost:%d", 0), - ) - By("sending alerts to each condition in the router") - for _, spec := range suiteSpec.specs { - err := alertingClient3.AlertClient().PostAlarm(context.TODO(), client.AlertObject{ - Id: spec.id, - Labels: map[string]string{ - ns: spec.id, - }, - Annotations: map[string]string{}, - }) - Expect(err).To(Succeed()) + err = router.UpdateEndpoint(server2.Endpoint().Id, server1.Endpoint()) + Expect(err).To(Succeed()) + for _, spec := range suiteSpec.specs { + if len(spec.servers) != 0 { + spec.servers[0] = server1 } - Eventually(func() error { - return suiteSpec.ExpectAlertsToBeRouted(amPort3) - }, time.Second*30, time.Second*1).Should(Succeed()) - ca3() - }) + } + + By("send an an alert to each specs") + amPort3, ca3 := alerting.RunAlertManager(env, router, tmpConfigDir, step+".yaml") + defer ca3() + alertingClient3 = client.NewClient( + nil, + fmt.Sprintf("http://localhost:%d", amPort3), + fmt.Sprintf("http://localhost:%d", 0), + ) + By("sending alerts to each condition in the router") + for _, spec := range suiteSpec.specs { + err := alertingClient3.AlertClient().PostAlarm(context.TODO(), client.AlertObject{ + Id: spec.id, + Labels: map[string]string{ + ns: spec.id, + }, + Annotations: map[string]string{}, + }) + Expect(err).To(Succeed()) + } + Eventually(func() error { + return suiteSpec.ExpectAlertsToBeRouted(amPort3) + }, time.Second*30, time.Second*1).Should(Succeed()) + ca3() }) }) -} +}) type testSpecSuite struct { name string diff --git a/test/plugins/alerting/slo_test.go b/test/plugins/alerting/slo_test.go index 756ffee1ee..6c61f78ec0 100644 --- a/test/plugins/alerting/slo_test.go +++ b/test/plugins/alerting/slo_test.go @@ -33,6 +33,7 @@ const ( var _ = Describe("SLO Alerting", Ordered, Label("integration"), func() { var sloClient slov1.SLOClient + var env *test.Environment BeforeAll(func() { env = &test.Environment{} Expect(env).NotTo(BeNil()) From 2da63c3fad23e41710a2b8a48f34573c6d293b6a Mon Sep 17 00:00:00 2001 From: Alexandre Lamarre Date: Thu, 14 Sep 2023 16:02:56 -0400 Subject: [PATCH 3/7] changes to SLO metric generation --- pkg/apis/slo/v1/slo.proto | 1 + pkg/metrics/compat/unmarshal.go | 5 +- pkg/metrics/compat/unmarshal_test.go | 6 +- pkg/slo/backend/metrics/manager.go | 12 +- test/plugins/metrics/slo_test.go | 174 +++++++++++++++++++-------- 5 files changed, 139 insertions(+), 59 deletions(-) diff --git a/pkg/apis/slo/v1/slo.proto b/pkg/apis/slo/v1/slo.proto index 051ed1be7b..3b8abfa6c6 100644 --- a/pkg/apis/slo/v1/slo.proto +++ b/pkg/apis/slo/v1/slo.proto @@ -267,6 +267,7 @@ message PlotVector { } message DataPoint { + // expected to be between 0 and 100 double sli = 1; google.protobuf.Timestamp timestamp = 2; } diff --git a/pkg/metrics/compat/unmarshal.go b/pkg/metrics/compat/unmarshal.go index d7364ad62b..5c75c1c646 100644 --- a/pkg/metrics/compat/unmarshal.go +++ b/pkg/metrics/compat/unmarshal.go @@ -97,12 +97,11 @@ func (qr *QueryResult) GetScalar() (*model.Scalar, error) { } type Sample struct { - Value float64 - // TODO Milliseconds? + Value float64 Timestamp int64 } -func (qr *QueryResult) MapToSamples() []Sample { +func (qr *QueryResult) LinearSamples() []Sample { res := []Sample{} switch qr.V.Type() { case model.ValVector: diff --git a/pkg/metrics/compat/unmarshal_test.go b/pkg/metrics/compat/unmarshal_test.go index e3f743145a..3eb20bc518 100644 --- a/pkg/metrics/compat/unmarshal_test.go +++ b/pkg/metrics/compat/unmarshal_test.go @@ -57,7 +57,7 @@ var _ = Describe("Prometheus Query response unmarshalling", Label("unit"), func( Expect(scalar.Value).To(Equal(scalarData.Value)) Expect(scalar.Timestamp).To(Equal(scalarData.Timestamp)) - samples := qr.MapToSamples() + samples := qr.LinearSamples() Expect(samples).To(HaveLen(1)) Expect(samples[0].Value).To(Equal(float64(scalarData.Value))) Expect(samples[0].Timestamp).To(Equal(int64(scalarData.Timestamp))) @@ -114,7 +114,7 @@ var _ = Describe("Prometheus Query response unmarshalling", Label("unit"), func( Expect(err).To(Succeed()) Expect(vector).NotTo(BeNil()) - samples := qr.MapToSamples() + samples := qr.LinearSamples() Expect(samples).To(HaveLen(2)) Expect(samples[0].Value).To(Equal(float64(vectorData[1].Value))) Expect(samples[0].Timestamp).To(Equal(int64(vectorData[1].Timestamp))) @@ -186,7 +186,7 @@ var _ = Describe("Prometheus Query response unmarshalling", Label("unit"), func( Expect(err).To(Succeed()) Expect(matrix).NotTo(BeNil()) - samples := qr.MapToSamples() + samples := qr.LinearSamples() Expect(samples).To(HaveLen(3)) Expect(samples[0].Value).To(Equal(float64(2))) Expect(samples[0].Timestamp).To(Equal(int64(0))) diff --git a/pkg/slo/backend/metrics/manager.go b/pkg/slo/backend/metrics/manager.go index 828f12593d..6a29ba30b7 100644 --- a/pkg/slo/backend/metrics/manager.go +++ b/pkg/slo/backend/metrics/manager.go @@ -215,7 +215,7 @@ func (m *MetricsSLOStore) Status(ctx context.Context, slo *slov1.SLOData) (*slov if err != nil { return nil, err } - if len(qrSLI.MapToSamples()) == 0 { + if len(qrSLI.LinearSamples()) == 0 { return &slov1.SLOStatus{ State: slov1.SLOStatusState_NoData, }, nil @@ -234,7 +234,7 @@ func (m *MetricsSLOStore) Status(ctx context.Context, slo *slov1.SLOData) (*slov if err != nil { return nil, err } - samples := qrBudget.MapToSamples() + samples := qrBudget.LinearSamples() if len(samples) == 0 { return &slov1.SLOStatus{ State: slov1.SLOStatusState_NoData, @@ -260,7 +260,7 @@ func (m *MetricsSLOStore) Status(ctx context.Context, slo *slov1.SLOData) (*slov if err != nil { return nil, err } - samples = qrPage.MapToSamples() + samples = qrPage.LinearSamples() if len(samples) != 0 { lastPageAlert := samples[len(samples)-1].Value if lastPageAlert > 0 { @@ -282,7 +282,7 @@ func (m *MetricsSLOStore) Status(ctx context.Context, slo *slov1.SLOData) (*slov if err != nil { return nil, err } - samples = qrTicket.MapToSamples() + samples = qrTicket.LinearSamples() if len(samples) != 0 { lastPageAlert := samples[len(samples)-1].Value if lastPageAlert > 0 { @@ -334,7 +334,7 @@ func (m *MetricsSLOStore) Preview(ctx context.Context, slo *slov1.CreateSLOReque return nil, err } - samples := qrSLI.MapToSamples() + samples := qrSLI.LinearSamples() plotVector := &slov1.PlotVector{ Objective: slo.Slo.Target.Value, @@ -343,7 +343,7 @@ func (m *MetricsSLOStore) Preview(ctx context.Context, slo *slov1.CreateSLOReque for i := 0; i < len(samples); i++ { plotVector.Items[i] = &slov1.DataPoint{ - Sli: samples[i].Value, + Sli: samples[i].Value * 100, Timestamp: timestamppb.Now(), } } diff --git a/test/plugins/metrics/slo_test.go b/test/plugins/metrics/slo_test.go index 1846d0aeba..be36163fc0 100644 --- a/test/plugins/metrics/slo_test.go +++ b/test/plugins/metrics/slo_test.go @@ -285,8 +285,7 @@ var _ = Describe("Prometheus SLOs", Ordered, Label("integration"), func() { "code", "200", "500", - 99.9, - 10000, + 50.0, ) Expect(err) }) @@ -351,42 +350,76 @@ var _ = Describe("Prometheus SLOs", Ordered, Label("integration"), func() { }) It("should be able to query the written data", func() { + // asStr := func(samples []compat.Sample) string { + // res := []string{} + // for _, s := range samples { + // res = append(res, fmt.Sprintf("V : %f, T : %d", s.Value, s.Timestamp)) + // } + // return strings.Join(res, " -- ") + // } resp, err := adminClient.QueryRange(env.Context(), &cortexadmin.QueryRangeRequest{ Tenants: []string{ "agent", }, - Query: "http_slo_response{code=\"200\"}", + Query: "sum(http_slo_response{code=\"200\"})", Start: timestamppb.New(time.Now().Add(-time.Hour)), End: timestamppb.New(time.Now()), Step: durationpb.New(time.Second * 1), }) Expect(err).To(Succeed()) Expect(resp).NotTo(BeNil()) - GinkgoWriter.Write(resp.Data) qr, err := compat.UnmarshalPrometheusResponse(resp.Data) Expect(err).To(Succeed()) - samples := qr.MapToSamples() + samples := qr.LinearSamples() + // GinkgoWriter.Write([]byte(asStr(samples))) Expect(samples).NotTo(HaveLen(0)) resp2, err := adminClient.QueryRange(env.Context(), &cortexadmin.QueryRangeRequest{ Tenants: []string{ "agent", }, - Query: "http_slo_response{code=\"500\"}", + Query: "sum(http_slo_response{code=~\"200|500\"})", Start: timestamppb.New(time.Now().Add(-time.Hour)), End: timestamppb.New(time.Now()), Step: durationpb.New(time.Second * 1), }) Expect(err).To(Succeed()) Expect(resp2).NotTo(BeNil()) - GinkgoWriter.Write(resp2.Data) qr2, err := compat.UnmarshalPrometheusResponse(resp2.Data) Expect(err).To(Succeed()) - samples2 := qr2.MapToSamples() + samples2 := qr2.LinearSamples() + // GinkgoWriter.Write([]byte(asStr(samples2))) Expect(samples2).NotTo(HaveLen(0)) + Eventually(func() error { + resp3, err := adminClient.QueryRange(env.Context(), &cortexadmin.QueryRangeRequest{ + Tenants: []string{ + "agent", + }, + Query: "sum(rate(http_slo_response{code=~\"200\"}[1m])) / sum(rate(http_slo_response{code=~\"500|200\"}[1m]))", + Start: timestamppb.New(time.Now().Add(-time.Hour)), + End: timestamppb.New(time.Now()), + Step: durationpb.New(time.Second * 1), + }) + if err != nil { + return err + } + if resp3 == nil { + return fmt.Errorf("nil response") + } + qr3, err := compat.UnmarshalPrometheusResponse(resp3.Data) + if err != nil { + return err + } + samples3 := qr3.LinearSamples() + // GinkgoWriter.Write([]byte(asStr(samples3))) + if len(samples3) == 0 { + return fmt.Errorf("no samples") + } + return nil + }).Should(Succeed()) }) - XIt("should create an SLO based on the written data", func() { + It("should create an SLO based on the written data", func() { req := &slov1.CreateSLORequest{ Slo: util.ProtoClone(metricsSlo), } @@ -397,14 +430,37 @@ var _ = Describe("Prometheus SLOs", Ordered, Label("integration"), func() { for _, item := range plotVec.PlotVector.Items { Expect(item).NotTo(BeNil()) - Expect(item.Sli).To(BeNumerically(">", float64(0))) //FIXME: it is 0 right now - Expect(item.Sli).To(BeNumerically("<", float64(100))) + Expect(item.Sli).To(BeNumerically(">", float64(0))) + Expect(item.Sli).To(BeNumerically("<=", float64(100))) } ref, err := sloDatasource.Create(env.Context(), req) Expect(err).To(Succeed()) Expect(ref).NotTo(BeNil()) Expect(ref.Id).NotTo(BeEmpty()) + + By("verifying it eventually has a status") + Eventually(func() error { + status, err := sloDatasource.Status(env.Context(), &slov1.SLOData{ + Id: ref.Id, + SLO: req.Slo, + }) + + if err != nil { + return err + } + + if status.State == slov1.SLOStatusState_Creating { + return fmt.Errorf("still creating") + } + if status.State == slov1.SLOStatusState_InternalError { + return fmt.Errorf("internal error") + } + if status.State == slov1.SLOStatusState_NoData { + return fmt.Errorf("no data") + } + return nil + }).Should(Succeed()) }) }) }) @@ -449,39 +505,11 @@ func writeSLO( period time.Duration, metricName, serviceName, eventName, goodEvent, totalEvent string, expectedSLI float64, - numSamples int, ) error { if expectedSLI > 100 && expectedSLI < 0 { return fmt.Errorf("expected SLI must be between 0 and 100") } - now := time.Now() - start := now.Add(-period) - - goodTs := []int64{} - goodSamples := int(float64(numSamples) * (expectedSLI / 100)) - goodStepDuration := int(period.Milliseconds()) / goodSamples - totalSamples := int(float64(numSamples) * ((100 - expectedSLI) / 100)) - totalStepDuration := int(period.Milliseconds()) / totalSamples - for i := 0; i < goodSamples; i++ { - toAdd := time.Duration(i*goodStepDuration) * time.Millisecond - goodTs = append(goodTs, start.Add(toAdd).UnixMilli()) - } - totalTs := []int64{} - for i := 0; i < totalSamples; i++ { - - toAdd := time.Duration(i*totalStepDuration) * time.Millisecond - totalTs = append(totalTs, start.Add(toAdd).UnixMilli()) - } - - if len(goodTs) == 0 { - panic("bug : test should have non-zero good event samples") - } - - if len(totalTs) == 0 { - panic("bug : test should have non-zero total event samples") - } - wreq := &prompb.WriteRequest{ Timeseries: []prompb.TimeSeries{}, } @@ -502,20 +530,72 @@ func writeSLO( }, } - for i, ts := range goodTs { + now := time.Now() + start := now.Add(-period) + + // 4 zeroes + precision := 1000 + iExpectedSLI := precision * int(expectedSLI) + iMax := 100 * precision + goodVal := 1 + totalVal := 1 + + // write every 250ms + for i := 0; i < int(period.Milliseconds()); i += 500 { + ts := start.Add(time.Duration(i) * time.Millisecond) + cur := i % iMax + if cur < iMax-iExpectedSLI { + totalVal += 1 + } else { + goodVal += 1 + } goodEvents.Samples = append(goodEvents.Samples, prompb.Sample{ - Value: float64(i + 1), - Timestamp: ts, + Value: float64(goodVal), + Timestamp: ts.UnixMilli(), }) - } - - for i, ts := range totalTs { totalEvents.Samples = append(totalEvents.Samples, prompb.Sample{ - Value: float64(i + 1), - Timestamp: ts, + Value: float64(totalVal), + Timestamp: ts.UnixMilli(), }) } + // goodSamples := int(float64(numSamples) * (expectedSLI / 100)) + // goodStepDuration := int(period.Milliseconds()) / goodSamples + // totalSamples := int(float64(numSamples) * ((100 - expectedSLI) / 100)) + // totalStepDuration := int(period.Milliseconds()) / totalSamples + // for i := 0; i < goodSamples; i++ { + // toAdd := time.Duration(i*goodStepDuration) * time.Millisecond + // goodTs = append(goodTs, start.Add(toAdd).UnixMilli()) + // } + // totalTs := []int64{} + // for i := 0; i < totalSamples; i++ { + + // toAdd := time.Duration(i*totalStepDuration) * time.Millisecond + // totalTs = append(totalTs, start.Add(toAdd).UnixMilli()) + // } + + if len(goodEvents.Samples) == 0 { + panic("bug : no good events") + } + + if len(totalEvents.Samples) == 0 { + panic("bug : no total events") + } + + // for i, ts := range goodTs { + // goodEvents.Samples = append(goodEvents.Samples, prompb.Sample{ + // Value: float64(i + 1), + // Timestamp: ts, + // }) + // } + + // for i, ts := range totalTs { + // totalEvents.Samples = append(totalEvents.Samples, prompb.Sample{ + // Value: float64(i + 1), + // Timestamp: ts, + // }) + // } + wreq.Timeseries = append(wreq.Timeseries, goodEvents) wreq.Timeseries = append(wreq.Timeseries, totalEvents) return remoteWrite(ctx, rw, wreq) From cc55985b4c1574ac76dce689befce6e7a25fc2cf Mon Sep 17 00:00:00 2001 From: Alexandre Lamarre Date: Mon, 18 Sep 2023 12:38:32 -0400 Subject: [PATCH 4/7] add mwmb alerts to prometheus gen --- pkg/slo/backend/metrics/manager.go | 39 +++++--- pkg/slo/backend/metrics/prometheus.go | 22 ++++- test/plugins/metrics/cortexadmin_test.go | 2 +- test/plugins/metrics/slo_test.go | 116 +++++++++++++++-------- 4 files changed, 123 insertions(+), 56 deletions(-) diff --git a/pkg/slo/backend/metrics/manager.go b/pkg/slo/backend/metrics/manager.go index 6a29ba30b7..0db66ff8ad 100644 --- a/pkg/slo/backend/metrics/manager.go +++ b/pkg/slo/backend/metrics/manager.go @@ -87,8 +87,22 @@ func (m *MetricsSLOStore) Create(ctx context.Context, req *slov1.CreateSLOReques func (m *MetricsSLOStore) Update(ctx context.Context, incoming, existing *slov1.SLOData) (*slov1.SLOData, error) { incomingSLO := backend.SLODataToStruct(incoming) existingSLO := backend.SLODataToStruct(existing) + if err := incomingSLO.Validate(); err != nil { + return nil, err + } + if err := existingSLO.Validate(); err != nil { + return nil, err + } + sloGen, err := NewSLOGenerator(*incomingSLO) + if err != nil { + return nil, err + } + rg, err := sloGen.AsRuleGroup() + if err != nil { + return nil, err + } - yamlBytes, err := yaml.Marshal(incomingSLO) + yamlBytes, err := yaml.Marshal(rg) if err != nil { return nil, err } @@ -247,20 +261,19 @@ func (m *MetricsSLOStore) Status(ctx context.Context, slo *slov1.SLOData) (*slov }, nil } - // 4. - pageAlert := sloGen.PageAlert().Expr() - rawPageAlert, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ + ticketAlert := sloGen.TicketAlert().Expr() + rawTicketAlert, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ Tenants: []string{clusterId}, - Query: pageAlert, + Query: ticketAlert, }) if err != nil { return nil, err } - qrPage, err := compat.UnmarshalPrometheusResponse(rawPageAlert.Data) + qrTicket, err := compat.UnmarshalPrometheusResponse(rawTicketAlert.Data) if err != nil { return nil, err } - samples = qrPage.LinearSamples() + samples = qrTicket.LinearSamples() if len(samples) != 0 { lastPageAlert := samples[len(samples)-1].Value if lastPageAlert > 0 { @@ -270,19 +283,20 @@ func (m *MetricsSLOStore) Status(ctx context.Context, slo *slov1.SLOData) (*slov } } - ticketAlert := sloGen.TicketAlert().Expr() - rawTicketAlert, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ + // 4. + pageAlert := sloGen.PageAlert().Expr() + rawPageAlert, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ Tenants: []string{clusterId}, - Query: ticketAlert, + Query: pageAlert, }) if err != nil { return nil, err } - qrTicket, err := compat.UnmarshalPrometheusResponse(rawTicketAlert.Data) + qrPage, err := compat.UnmarshalPrometheusResponse(rawPageAlert.Data) if err != nil { return nil, err } - samples = qrTicket.LinearSamples() + samples = qrPage.LinearSamples() if len(samples) != 0 { lastPageAlert := samples[len(samples)-1].Value if lastPageAlert > 0 { @@ -291,6 +305,7 @@ func (m *MetricsSLOStore) Status(ctx context.Context, slo *slov1.SLOData) (*slov }, nil } } + return &slov1.SLOStatus{ State: slov1.SLOStatusState_Ok, }, nil diff --git a/pkg/slo/backend/metrics/prometheus.go b/pkg/slo/backend/metrics/prometheus.go index 61d3352803..8c34aac0ce 100644 --- a/pkg/slo/backend/metrics/prometheus.go +++ b/pkg/slo/backend/metrics/prometheus.go @@ -255,11 +255,29 @@ func (s *SLOGeneratorImpl) ErrorBudgetRemaining() MetricGenerator { } func (s *SLOGeneratorImpl) PageAlert() MetricGenerator { - panic("not implemented") // TODO: Implement + return NewMWMBAlertGenerator( + "slo:mwmb_alert:page", + s.slo.IdLabels, + s.windows.PageQuick, + s.windows.PageSlow, + func(window prommodel.Duration) MetricGenerator { + return s.SLI(window) + }, + s.options, + ) } func (s *SLOGeneratorImpl) TicketAlert() MetricGenerator { - panic("not implemented") // TODO: Implement + return NewMWMBAlertGenerator( + "slo:mwmb_alert:ticket", + s.slo.IdLabels, + s.windows.TicketQuick, + s.windows.TicketSlow, + func(window prommodel.Duration) MetricGenerator { + return s.SLI(window) + }, + s.options, + ) } var _ MWMBSLOGenerator = (*SLOGeneratorImpl)(nil) diff --git a/test/plugins/metrics/cortexadmin_test.go b/test/plugins/metrics/cortexadmin_test.go index eafee3f8a9..6b3b5d767f 100644 --- a/test/plugins/metrics/cortexadmin_test.go +++ b/test/plugins/metrics/cortexadmin_test.go @@ -106,7 +106,7 @@ func setMockKubernetesPodState(kubePort int, pod *mockPod) { }() } -var _ = Describe("Converting ServiceLevelObjective Messages to Prometheus Rules", Ordered, Label("integration", "slow"), func() { +var _ = XDescribe("Converting ServiceLevelObjective Messages to Prometheus Rules", Ordered, Label("integration", "slow"), func() { ctx := context.Background() var env *test.Environment var adminClient cortexadmin.CortexAdminClient diff --git a/test/plugins/metrics/slo_test.go b/test/plugins/metrics/slo_test.go index be36163fc0..94f02f0763 100644 --- a/test/plugins/metrics/slo_test.go +++ b/test/plugins/metrics/slo_test.go @@ -30,6 +30,8 @@ import ( "github.com/rancher/opni/pkg/util" "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" "github.com/rancher/opni/plugins/metrics/apis/cortexops" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/emptypb" "google.golang.org/protobuf/types/known/timestamppb" @@ -47,6 +49,7 @@ var _ = Describe("Prometheus SLOs", Ordered, Label("integration"), func() { var sloGen *metrics.SLOGeneratorImpl var sloDatasource backend.SLODatasource var rw remote.WriteClient + var sloRef *corev1.Reference BeforeAll(func() { env = &test.Environment{} Expect(env.Start()).To(Succeed()) @@ -285,7 +288,7 @@ var _ = Describe("Prometheus SLOs", Ordered, Label("integration"), func() { "code", "200", "500", - 50.0, + 99.0, ) Expect(err) }) @@ -435,31 +438,91 @@ var _ = Describe("Prometheus SLOs", Ordered, Label("integration"), func() { } ref, err := sloDatasource.Create(env.Context(), req) + sloRef = ref Expect(err).To(Succeed()) Expect(ref).NotTo(BeNil()) Expect(ref.Id).NotTo(BeEmpty()) By("verifying it eventually has a status") - Eventually(func() error { + Eventually(func() slov1.SLOStatusState { status, err := sloDatasource.Status(env.Context(), &slov1.SLOData{ Id: ref.Id, SLO: req.Slo, }) if err != nil { - return err + return slov1.SLOStatusState_InProgress } - if status.State == slov1.SLOStatusState_Creating { - return fmt.Errorf("still creating") - } - if status.State == slov1.SLOStatusState_InternalError { - return fmt.Errorf("internal error") + return status.State + }, time.Second*5, time.Millisecond*100).Should( + Equal(slov1.SLOStatusState_Ok), + ) + + By("verifying after update to a higher target, it should have breached the error budget") + incoming := &slov1.SLOData{ + Id: ref.Id, + SLO: util.ProtoClone(metricsSlo), + } + existing := &slov1.SLOData{ + Id: ref.Id, + SLO: util.ProtoClone(metricsSlo), + } + incoming.SLO.Target.Value = 99.9999 + _, err = sloDatasource.Update(env.Context(), incoming, existing) + Expect(err).To(Succeed()) + + Eventually(func() slov1.SLOStatusState { + status, err := sloDatasource.Status(env.Context(), incoming) + if err != nil { + return slov1.SLOStatusState_InProgress } - if status.State == slov1.SLOStatusState_NoData { - return fmt.Errorf("no data") + return status.State + }).Should(Equal(slov1.SLOStatusState_Breaching)) + + // TODO : this is wrong, might need to tweak alert rules + // By("verifying updates of a target that should pick up burn rate") + + // incoming2 := util.ProtoClone(incoming) + // incoming2.SLO.Target.Value = 99.5 + // _, err = sloDatasource.Update(env.Context(), incoming2, incoming) + // Expect(err).To(Succeed()) + + // Eventually(func() slov1.SLOStatusState { + // status, err := sloDatasource.Status(env.Context(), incoming2) + // if err != nil { + // return slov1.SLOStatusState_InProgress + // } + // return status.State + // }).Should(Equal(slov1.SLOStatusState_Warning)) + + }) + + It("should be able to delete a metrics SLO", func() { + Expect(sloRef).NotTo(BeNil()) + Expect(sloRef.Id).NotTo(BeEmpty()) + Expect(sloDatasource.Delete(env.Context(), &slov1.SLOData{ + Id: sloRef.Id, + SLO: util.ProtoClone(metricsSlo), + })).To(Succeed()) + + Eventually(func() error { + _, err := adminClient.GetRule(env.Context(), &cortexadmin.GetRuleRequest{ + ClusterId: "agent", + Namespace: "slo", + GroupName: sloRef.Id, + }) + if err == nil { + return fmt.Errorf("rule still exists") + } + if st, ok := status.FromError(err); ok && st.Code() == codes.NotFound { + return nil + } else if ok { + return fmt.Errorf("unexpected status code %d", st.Code()) + } else if err != nil { + return err } - return nil + return fmt.Errorf("unreachable code?") }).Should(Succeed()) }) }) @@ -559,21 +622,6 @@ func writeSLO( }) } - // goodSamples := int(float64(numSamples) * (expectedSLI / 100)) - // goodStepDuration := int(period.Milliseconds()) / goodSamples - // totalSamples := int(float64(numSamples) * ((100 - expectedSLI) / 100)) - // totalStepDuration := int(period.Milliseconds()) / totalSamples - // for i := 0; i < goodSamples; i++ { - // toAdd := time.Duration(i*goodStepDuration) * time.Millisecond - // goodTs = append(goodTs, start.Add(toAdd).UnixMilli()) - // } - // totalTs := []int64{} - // for i := 0; i < totalSamples; i++ { - - // toAdd := time.Duration(i*totalStepDuration) * time.Millisecond - // totalTs = append(totalTs, start.Add(toAdd).UnixMilli()) - // } - if len(goodEvents.Samples) == 0 { panic("bug : no good events") } @@ -582,20 +630,6 @@ func writeSLO( panic("bug : no total events") } - // for i, ts := range goodTs { - // goodEvents.Samples = append(goodEvents.Samples, prompb.Sample{ - // Value: float64(i + 1), - // Timestamp: ts, - // }) - // } - - // for i, ts := range totalTs { - // totalEvents.Samples = append(totalEvents.Samples, prompb.Sample{ - // Value: float64(i + 1), - // Timestamp: ts, - // }) - // } - wreq.Timeseries = append(wreq.Timeseries, goodEvents) wreq.Timeseries = append(wreq.Timeseries, totalEvents) return remoteWrite(ctx, rw, wreq) @@ -641,7 +675,7 @@ var ( }, }, SloPeriod: "2h", - Target: &slov1.Target{Value: 99.9}, + Target: &slov1.Target{Value: 90.0}, BudgetingInterval: durationpb.New(time.Second * 1), } ) From 74e122e7b82e32f9ccc8cb0349326f7cc0d142a7 Mon Sep 17 00:00:00 2001 From: Alexandre Lamarre Date: Mon, 18 Sep 2023 17:19:47 -0400 Subject: [PATCH 5/7] purge2 --- internal/cmd/testenv/main.go | 5 - pkg/slo/backend/slo_test.go | 3 - pkg/slo/query/collectors.go | 68 --- pkg/slo/query/def.go | 330 --------------- pkg/slo/query/histogram.go | 71 ---- pkg/slo/query/matchers.go | 23 - pkg/slo/query/opensearch.go | 6 - pkg/slo/query/prometheus.go | 69 --- pkg/slo/query/query_suite_test.go | 14 - pkg/slo/query/query_test.go | 75 ---- pkg/slo/query/ratio.go | 75 ---- pkg/slo/query/validate.go | 36 -- pkg/test/environment.go | 139 ------ test/plugins/metrics/cortex_query_test.go | 2 + test/plugins/metrics/cortexadmin_test.go | 490 ---------------------- 15 files changed, 2 insertions(+), 1404 deletions(-) delete mode 100644 pkg/slo/backend/slo_test.go delete mode 100644 pkg/slo/query/collectors.go delete mode 100644 pkg/slo/query/def.go delete mode 100644 pkg/slo/query/histogram.go delete mode 100644 pkg/slo/query/matchers.go delete mode 100644 pkg/slo/query/opensearch.go delete mode 100644 pkg/slo/query/prometheus.go delete mode 100644 pkg/slo/query/query_suite_test.go delete mode 100644 pkg/slo/query/query_test.go delete mode 100644 pkg/slo/query/ratio.go delete mode 100644 pkg/slo/query/validate.go delete mode 100644 test/plugins/metrics/cortexadmin_test.go diff --git a/internal/cmd/testenv/main.go b/internal/cmd/testenv/main.go index e4222159d2..ef09bea240 100644 --- a/internal/cmd/testenv/main.go +++ b/internal/cmd/testenv/main.go @@ -189,11 +189,6 @@ func main() { c := make(chan os.Signal, 2) var closeOnce sync.Once signal.Notify(c, os.Interrupt) - iPort, _ = environment.StartInstrumentationServer() - kPort = environment.StartMockKubernetesMetricServer() - for i := 0; i < 100; i++ { - environment.SimulateKubeObject(kPort) - } testlog.Log.Infof(chalk.Green.Color("Instrumentation server listening on %d"), iPort) var client managementv1.ManagementClient if enableGateway { diff --git a/pkg/slo/backend/slo_test.go b/pkg/slo/backend/slo_test.go deleted file mode 100644 index d02f65a55f..0000000000 --- a/pkg/slo/backend/slo_test.go +++ /dev/null @@ -1,3 +0,0 @@ -package backend_test - -// TODO : diff --git a/pkg/slo/query/collectors.go b/pkg/slo/query/collectors.go deleted file mode 100644 index a8a27f8f63..0000000000 --- a/pkg/slo/query/collectors.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Module for defining collectors and their good/bad events API. - -There are registered manually in preconfigured metrics builder in `def.go` - -There are registered automatically to an auto-instrumentation server -in `pkg/test/instrumentation.go` -*/ -package query - -import ( - "fmt" - "math/rand" - "net/http" - - "github.com/prometheus/client_golang/prometheus" -) - -var verbs = []string{"POST", "GET", "PUT", "DELETE"} -var goodCodes = []int{200, 201, 202} -var badCodes = []int{404, 429, 500, 502, 503} - -var ( - uptimeCollector *prometheus.GaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "uptime_good", - }, []string{"hostname", "ip", "job"}) - - uptimeGoodEvents http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - uptimeCollector.WithLabelValues(r.Host, r.RemoteAddr, MockTestServerName).Set(1) - } - uptimeBadEvents http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - uptimeCollector.WithLabelValues(r.Host, r.RemoteAddr, MockTestServerName).Set(0) - } - - availabilityCollector *prometheus.CounterVec = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "http_request_duration_seconds_count", - }, - []string{"code", "verb"}, - ) - availabilityGoodEvents http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - randomStatusCode := goodCodes[rand.Intn(len(goodCodes))] - randomVerb := verbs[rand.Intn(len(verbs))] - - availabilityCollector.WithLabelValues(fmt.Sprintf("%d", randomStatusCode), randomVerb).Inc() - } - - availabilityBadEvents http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - randomStatusCode := badCodes[rand.Intn(len(badCodes))] - randomVerb := verbs[rand.Intn(len(verbs))] - availabilityCollector.WithLabelValues(fmt.Sprintf("%d", randomStatusCode), randomVerb).Inc() - } - - latencyCollector *prometheus.HistogramVec = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Name: "http_request_duration_seconds_bucket", - }, []string{"hostname", "ip", "job"}) - - latencyGoodEvents http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - // request duration faster than 0.3s / 0.300 ms - randLatency := rand.Float64() * 0.29 - latencyCollector.WithLabelValues(r.Host, r.RemoteAddr, MockTestServerName).Observe(randLatency) - } - - latencyBadEvents http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - // request duration slower than 0.3s / 0.300 ms - randLatency := rand.Float64() + 0.30 - latencyCollector.WithLabelValues(r.Host, r.RemoteAddr, MockTestServerName).Observe(randLatency) - } -) diff --git a/pkg/slo/query/def.go b/pkg/slo/query/def.go deleted file mode 100644 index 3959b10c2a..0000000000 --- a/pkg/slo/query/def.go +++ /dev/null @@ -1,330 +0,0 @@ -package query - -/* -This module defines the datatypes & interfaces used to define SLO queries. -`AvailableQueries` contains the list of all preconfigured queries. - -Queries used by SLOs must follow a format like : - -totalQueryTempl = template.Must(template.New("").Parse(` - sum(rate({{.MetricId}}{job="{{.JobId}}"}[{{"{{.window}}"}}])) - `)) -goodQueryTempl = template.Must(template.New("").Parse(` - sum(rate({{.MetricId}}{job="{{.JobId}}", {{.Filter}}}[{{"{{.window}}"}}])) -`)) - -Must : - -1. Include a nested template with a {{.window}} for SLOs to fill in -2. the templates must only include information that can be filled with the `templateExecutor` struct. - Note: templates are intended to be filled with *api.Service protobuf definitions, so expect only that information will - be available, when SLOs are created at runtime -*/ - -import ( - "net/http" - "regexp" - "strings" - "text/template" - - "github.com/prometheus/client_golang/prometheus" - "github.com/rancher/opni/pkg/slo/shared" -) - -var ( - AvailableQueries = make(map[string]MetricQuery) - GetDownstreamMetricQueryTempl = template.Must(template.New("").Parse(` - group by(__name__)({__name__=~"{{.NameRegex}}"}) - `)) -) - -type templateExecutor struct { - MetricIdGood string - MetricIdTotal string - JobId string -} - -// must not contain spaces -const MockTestServerName = "MyServer" - -func init() { - // Names should be unique for each pre-configured query, as they are used as keys - // in the map - - //FIXME: doesn't turn into a prometheus range - uptimeSLOQuery := New(). - Name("uptime"). - GoodQuery( - NewQueryBuilder(). - Query(` - (sum(rate({{.MetricIdGood}}{job="{{.JobId}}"} == 1)))[{{"{{.window}}"}}] - `). - MetricFilter(`up`). - BuildRatio()). - TotalQuery( - NewQueryBuilder(). - Query(` - (sum(rate({{.MetricIdTotal}}{job="{{.JobId}}"})))[{{"{{.window}}"}}] - `). - MetricFilter(`up`).BuildRatio()). - Collector(uptimeCollector). - GoodEventGenerator(uptimeGoodEvents). - BadEventGenerator(uptimeBadEvents). - Description("Measures the uptime of a kubernetes service"). - Datasource(shared.MonitoringDatasource).Build() - AvailableQueries[uptimeSLOQuery.name] = &uptimeSLOQuery - - httpAvailabilitySLOQuery := New(). - Name("http-availability"). - GoodQuery( - NewQueryBuilder(). - Query(` - (sum(rate({{.MetricIdGood}}{job="{{.JobId}}",code=~"(2..|3..)"}[{{"{{.window}}"}}]))) - `). - MetricFilter(`.*http_request_duration_seconds_count`). - BuildRatio()). - TotalQuery( - NewQueryBuilder(). - Query(` - (sum(rate({{.MetricIdTotal}}{job="{{.JobId}}"}[{{"{{.window}}"}}]))) - `). - MetricFilter(`.*http_request_duration_seconds_count`).BuildRatio()). - Collector(availabilityCollector). - GoodEventGenerator(availabilityGoodEvents). - BadEventGenerator(availabilityBadEvents). - Description(`Measures the availability of a kubernetes service using http status codes. - Codes 2XX and 3XX are considered as available.`). - Datasource(shared.MonitoringDatasource).Build() - AvailableQueries[httpAvailabilitySLOQuery.name] = &httpAvailabilitySLOQuery - - httpResponseTimeSLOQuery := New(). - Name("http-latency"). - GoodQuery( - NewQueryBuilder(). - Query(` - sum(rate({{.MetricIdGood}}{job="{{.JobId}},"le="0.3",verb!="WATCH"}[{{"{{.window}}"}}])) - `). - MetricFilter(`.*http_request_duration_seconds_bucket`). - BuildHistogram()). - TotalQuery( - NewQueryBuilder(). - Query(` - sum(rate({{.MetricIdTotal}}{job="{{.JobId}}",verb!="WATCH"}[{{"{{.window}}"}}])) - `). - MetricFilter(`.*http_request_duration_seconds_count`).BuildRatio()). - Collector(latencyCollector). - GoodEventGenerator(latencyGoodEvents). - BadEventGenerator(latencyBadEvents). - Description(`Quantifies the latency of http requests made against a kubernetes service - by classifying them as good (<=300ms) or bad(>=300ms)`). - Datasource(shared.MonitoringDatasource).Build() - AvailableQueries[httpResponseTimeSLOQuery.name] = &httpResponseTimeSLOQuery -} - -type matcher func([]string) string - -type SLOQueryResult struct { - GoodQuery string - TotalQuery string -} - -type ServiceInfo interface { - GetMetricIdGood() string - GetMetricIdTotal() string - GetJobId() string -} - -type MetricQuery interface { - // User facing name of the pre-confured metric - Name() string - // User-facing description of the pre-configured metric - Description() string - // Each metric has a unique opni datasource (monitoring vs logging) by which it is filtered by - Datasource() string - Construct(ServiceInfo) (*SLOQueryResult, error) - // Some metrics will have different labels for metrics, so handle them independently - GetGoodQuery() Query - GetTotalQuery() Query - - // Auto-instrumentation server methods - - GetCollector() prometheus.Collector - GetGoodEventGenerator() func(w http.ResponseWriter, r *http.Request) - GetBadEventGenerator() func(w http.ResponseWriter, r *http.Request) -} - -type Query interface { - FillQueryTemplate(info templateExecutor) (string, error) - GetMetricFilter() string - Validate() error - IsRatio() bool - BestMatch([]string) string - IsHistogram() bool - Construct(ServiceInfo) (string, error) -} - -type QueryBuilder interface { - Query(string) QueryBuilder - MetricFilter(string) QueryBuilder - Matcher(*matcher) QueryBuilder - BuildRatio() RatioQuery - BuildHistogram() HistogramQuery -} - -type queryBuilder struct { - query template.Template - filter regexp.Regexp - matcher matcher -} - -func NewQueryBuilder() QueryBuilder { - return queryBuilder{} -} - -func (q queryBuilder) Query(query string) QueryBuilder { - tmpl := template.Must(template.New("").Parse(strings.TrimSpace(query))) - q.query = *tmpl - return q -} - -func (q queryBuilder) MetricFilter(filter string) QueryBuilder { - regex := regexp.MustCompile(strings.TrimSpace(filter)) - q.filter = *regex - return q -} - -// defaults to `MatchMinLength` -func (q queryBuilder) Matcher(matcher *matcher) QueryBuilder { - if matcher == nil { - q.matcher = MatchMinLength - } else { - q.matcher = *matcher - } - return q -} - -func (q queryBuilder) BuildRatio() RatioQuery { - if q.matcher == nil { - q.matcher = MatchMinLength - } - r := RatioQuery{ - query: q.query, - metricFilter: q.filter, - matcher: q.matcher, - } - err := r.Validate() - if err != nil { - panic(err) - } - return r -} - -func (q queryBuilder) BuildHistogram() HistogramQuery { - if q.matcher == nil { - q.matcher = MatchMinLength - } - h := HistogramQuery{ - query: q.query, - metricFilter: q.filter, - matcher: q.matcher, - } - err := h.Validate() - if err != nil { - panic(err) - } - return h -} - -type SloQueryBuilder interface { - Name(name string) SloQueryBuilder - GoodQuery(q Query) SloQueryBuilder - TotalQuery(q Query) SloQueryBuilder - Description(description string) SloQueryBuilder - Datasource(datasource string) SloQueryBuilder - Collector(prometheus.Collector) SloQueryBuilder - GoodEventGenerator(goodEvents func(w http.ResponseWriter, r *http.Request)) SloQueryBuilder - BadEventGenerator(badEvents func(w http.ResponseWriter, r *http.Request)) SloQueryBuilder - Build() PrometheusQueryImpl -} - -type sloQueryBuilder struct { - name string - metricFilter regexp.Regexp - goodQuery Query - totalQuery Query - description string - datasource string - collector prometheus.Collector - totalCollector prometheus.Collector - goodEventsGen func(w http.ResponseWriter, r *http.Request) - badEventsGen func(w http.ResponseWriter, r *http.Request) -} - -func New() SloQueryBuilder { - return sloQueryBuilder{} -} - -func (s sloQueryBuilder) Name(name string) SloQueryBuilder { - s.name = name - return s -} - -func (s sloQueryBuilder) GoodQuery(q Query) SloQueryBuilder { - s.goodQuery = q - return s -} - -func (s sloQueryBuilder) TotalQuery(q Query) SloQueryBuilder { - s.totalQuery = q - return s -} - -func (s sloQueryBuilder) Description(description string) SloQueryBuilder { - s.description = description - return s -} - -func (s sloQueryBuilder) Datasource(datasource string) SloQueryBuilder { - s.datasource = datasource - return s -} - -func (s sloQueryBuilder) Collector( - collector prometheus.Collector) SloQueryBuilder { - s.collector = collector - return s -} - -func (s sloQueryBuilder) GoodEventGenerator( - goodEvents func(w http.ResponseWriter, r *http.Request), -) SloQueryBuilder { - s.goodEventsGen = goodEvents - return s -} - -func (s sloQueryBuilder) BadEventGenerator( - badEvents func(w http.ResponseWriter, r *http.Request), -) SloQueryBuilder { - s.badEventsGen = badEvents - return s -} - -func (s sloQueryBuilder) Build() PrometheusQueryImpl { - //TODO figure out how to validate at compile time - if s.collector == nil { - panic("auto-instrumentation collectors must be set") - } - if s.goodEventsGen == nil || s.badEventsGen == nil { - panic("auto-instrumentation events must be set") - } - return PrometheusQueryImpl{ - name: s.name, - datasource: s.datasource, - description: s.description, - GoodQuery: s.goodQuery, - TotalQuery: s.totalQuery, - collector: s.collector, - goodEventsGen: s.goodEventsGen, - badEventsGen: s.badEventsGen, - } -} diff --git a/pkg/slo/query/histogram.go b/pkg/slo/query/histogram.go deleted file mode 100644 index ab2690d3e4..0000000000 --- a/pkg/slo/query/histogram.go +++ /dev/null @@ -1,71 +0,0 @@ -package query - -import ( - "bytes" - "net/http" - "regexp" - "strings" - "text/template" - - "github.com/prometheus/client_golang/prometheus" -) - -type HistogramQuery struct { - query template.Template - metricFilter regexp.Regexp - matcher matcher - goodCollector *prometheus.Collector - totalCollector *prometheus.Collector - goodEvents func(w http.ResponseWriter, r *http.Request) - totalEvents func(w http.ResponseWriter, r *http.Request) -} - -func (hq HistogramQuery) GetGoodCollector() prometheus.Collector { - return *hq.goodCollector -} - -func (hq HistogramQuery) GetTotalCollector() prometheus.Collector { - return *hq.totalCollector -} -func (hq HistogramQuery) GetGoodEvents() func(w http.ResponseWriter, r *http.Request) { - return hq.goodEvents -} -func (hq HistogramQuery) GetTotalEvents() func(w http.ResponseWriter, r *http.Request) { - return hq.totalEvents -} - -func (hq HistogramQuery) FillQueryTemplate(info templateExecutor) (string, error) { - var query bytes.Buffer - if err := hq.query.Execute(&query, info); err != nil { - return "", err - } - return strings.TrimSpace(query.String()), nil -} - -func (hq HistogramQuery) GetMetricFilter() string { - return hq.metricFilter.String() -} - -func (hq HistogramQuery) Validate() error { - return validatePromQl(hq.query) -} - -func (hq HistogramQuery) IsRatio() bool { - return false -} - -func (hq HistogramQuery) IsHistogram() bool { - return true -} - -func (hq HistogramQuery) Construct(serv ServiceInfo) (string, error) { - return hq.FillQueryTemplate(templateExecutor{ - MetricIdGood: serv.GetMetricIdGood(), - MetricIdTotal: serv.GetMetricIdTotal(), - JobId: serv.GetJobId(), - }) -} - -func (hq HistogramQuery) BestMatch(in []string) string { - return hq.matcher(in) -} diff --git a/pkg/slo/query/matchers.go b/pkg/slo/query/matchers.go deleted file mode 100644 index ddd85ea22d..0000000000 --- a/pkg/slo/query/matchers.go +++ /dev/null @@ -1,23 +0,0 @@ -package query - -import "math" - -/* -This module defines matcher functions for finding the best match for the -discovered downstream metrics -*/ - -func MatchMinLength(metrics []string) string { - min := math.MaxInt // largest int - metricId := "" - for _, m := range metrics { - if m == "" { - continue - } - if len(m) < min { - min = len(m) - metricId = m - } - } - return metricId -} diff --git a/pkg/slo/query/opensearch.go b/pkg/slo/query/opensearch.go deleted file mode 100644 index 0100a189cb..0000000000 --- a/pkg/slo/query/opensearch.go +++ /dev/null @@ -1,6 +0,0 @@ -package query - -/* -Make sure pre-configured metrics can be exported as valid opensearch / monitors -to the SLO api. -*/ diff --git a/pkg/slo/query/prometheus.go b/pkg/slo/query/prometheus.go deleted file mode 100644 index 4da7679c15..0000000000 --- a/pkg/slo/query/prometheus.go +++ /dev/null @@ -1,69 +0,0 @@ -package query - -/* -Make sure pre-configured metrics can be exported as valid prometheus/ promql -to the SLO api. -*/ - -import ( - "net/http" - "regexp" - - "github.com/prometheus/client_golang/prometheus" -) - -type PrometheusQueryImpl struct { - name string - GoodQuery Query - TotalQuery Query - LabelRegex regexp.Regexp - datasource string - description string - collector prometheus.Collector - goodEventsGen func(w http.ResponseWriter, r *http.Request) - badEventsGen func(w http.ResponseWriter, r *http.Request) -} - -// The actual metricId and window are only known at SLO creation time -func (p *PrometheusQueryImpl) Construct(service ServiceInfo) (*SLOQueryResult, error) { - goodQueryStr, err := p.GoodQuery.Construct(service) - if err != nil { - return nil, err - } - totalQueryStr, err := p.TotalQuery.Construct(service) - if err != nil { - return nil, err - } - return &SLOQueryResult{GoodQuery: goodQueryStr, TotalQuery: totalQueryStr}, nil -} - -func (p *PrometheusQueryImpl) Datasource() string { - return p.datasource -} - -func (p *PrometheusQueryImpl) Description() string { - return p.description -} - -func (p *PrometheusQueryImpl) Name() string { - return p.name -} - -func (p *PrometheusQueryImpl) GetGoodQuery() Query { - return p.GoodQuery -} - -func (p *PrometheusQueryImpl) GetTotalQuery() Query { - return p.TotalQuery -} - -func (p *PrometheusQueryImpl) GetCollector() prometheus.Collector { - return p.collector -} - -func (p *PrometheusQueryImpl) GetGoodEventGenerator() func(w http.ResponseWriter, r *http.Request) { - return p.goodEventsGen -} -func (p *PrometheusQueryImpl) GetBadEventGenerator() func(w http.ResponseWriter, r *http.Request) { - return p.badEventsGen -} diff --git a/pkg/slo/query/query_suite_test.go b/pkg/slo/query/query_suite_test.go deleted file mode 100644 index 9a6ffd5be7..0000000000 --- a/pkg/slo/query/query_suite_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package query_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - _ "github.com/rancher/opni/pkg/test/setup" -) - -func TestQuery(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Query Suite") -} diff --git a/pkg/slo/query/query_test.go b/pkg/slo/query/query_test.go deleted file mode 100644 index d1b10dc336..0000000000 --- a/pkg/slo/query/query_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package query_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - api "github.com/rancher/opni/pkg/apis/slo/v1" - "github.com/rancher/opni/pkg/slo/query" -) - -func constructionShouldSucceed(q *query.SLOQueryResult, err error) { - Expect(err).To(Succeed()) - Expect(q).To(Not(BeNil())) - Expect(q.GoodQuery).To(Not(BeNil())) - Expect(q.TotalQuery).To(Not(BeNil())) -} - -var _ = Describe("Converting ServiceLevelObjective Messages to Prometheus Rules", Ordered, Label("unit", "slow"), func() { - // ctx := context.Background() - - validMetricQuery := func(query query.MetricQuery) error { - return nil - } - - When("We define pre-configured metrics in def.go", func() { - It("Should produce valid templates and regex at compile time", func() { - Expect(query.AvailableQueries).ToNot(HaveLen(0)) // > 0 available pre configured metrics - }) - - It("Query implementations should implement the interfaces required", func() { - q := &query.PrometheusQueryImpl{} - validMetricQuery(q) - }) - }) - - When("We use matchers", func() { - It("Should use a MatchMinLength to match the shortest metric", func() { - m := query.MatchMinLength - discoveredMetrics := []string{ - "alertmanager_http_request_duration_seconds_bucket", - "alertmanager_http_request_duration_seconds_count", - "alertmanager_http_request_duration_seconds_sum", - } - Expect(m(discoveredMetrics)).To(Equal("alertmanager_http_request_duration_seconds_sum")) - }) - }) - - When("We construct metrics via Service defintions", func() { - It("Should fill in the templates & return a response struct", func() { - uptime, ok := query.AvailableQueries["uptime"] - Expect(ok).To(BeTrue()) - service := &api.ServiceInfo{ - JobId: "prometheus", - MetricName: "uptime", - MetricIdGood: "up", - MetricIdTotal: "up", - ClusterId: "agent", - } - respUp, err := uptime.Construct(service) - constructionShouldSucceed(respUp, err) - Expect(respUp.GoodQuery).To(Equal("(sum(rate(up{job=\"prometheus\"} == 1)))[{{.window}}]")) - Expect(respUp.TotalQuery).To(Equal("(sum(rate(up{job=\"prometheus\"})))[{{.window}}]")) - - latency, ok := query.AvailableQueries["http-latency"] - Expect(ok).To(BeTrue()) - respLa, err := latency.Construct(service) - constructionShouldSucceed(respLa, err) - - availability, ok := query.AvailableQueries["http-availability"] - Expect(ok).To(BeTrue()) - respAv, err := availability.Construct(service) - constructionShouldSucceed(respAv, err) - - }) - }) -}) diff --git a/pkg/slo/query/ratio.go b/pkg/slo/query/ratio.go deleted file mode 100644 index 0cb2d8b55a..0000000000 --- a/pkg/slo/query/ratio.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Contains implementation details for slo ratio query metrics -*/ - -package query - -import ( - "bytes" - "net/http" - "regexp" - "strings" - "text/template" - - "github.com/prometheus/client_golang/prometheus" -) - -type RatioQuery struct { - query template.Template - metricFilter regexp.Regexp // how to assign the metric to the query - matcher matcher - goodCollector *prometheus.Collector - totalCollector *prometheus.Collector - goodEvents func(w http.ResponseWriter, r *http.Request) - totalEvents func(w http.ResponseWriter, r *http.Request) -} - -func (rq RatioQuery) GetGoodCollector() prometheus.Collector { - return *rq.goodCollector -} - -func (rq RatioQuery) GetTotalCollector() prometheus.Collector { - return *rq.totalCollector -} -func (rq RatioQuery) GetGoodEvents() func(w http.ResponseWriter, r *http.Request) { - return rq.goodEvents -} -func (rq RatioQuery) GetTotalEvents() func(w http.ResponseWriter, r *http.Request) { - return rq.totalEvents -} - -func (rq RatioQuery) FillQueryTemplate(info templateExecutor) (string, error) { - var query bytes.Buffer - if err := rq.query.Execute(&query, info); err != nil { - return "", err - } - return strings.TrimSpace(query.String()), nil -} - -func (rq RatioQuery) GetMetricFilter() string { - return rq.metricFilter.String() -} - -func (rq RatioQuery) Validate() error { - return validatePromQl(rq.query) -} - -func (rq RatioQuery) IsRatio() bool { - return true -} - -func (rq RatioQuery) IsHistogram() bool { - return false -} - -func (rq RatioQuery) Construct(serv ServiceInfo) (string, error) { - return rq.FillQueryTemplate(templateExecutor{ - MetricIdGood: serv.GetMetricIdGood(), - MetricIdTotal: serv.GetMetricIdTotal(), - JobId: serv.GetJobId(), - }) -} - -func (rq RatioQuery) BestMatch(in []string) string { - return rq.matcher(in) -} diff --git a/pkg/slo/query/validate.go b/pkg/slo/query/validate.go deleted file mode 100644 index 925cb71215..0000000000 --- a/pkg/slo/query/validate.go +++ /dev/null @@ -1,36 +0,0 @@ -package query - -import ( - "text/template" - // prommodel "github.com/prometheus/common/model" -) - -// fill in query templates with mock information -var mockTemplateExecutor = templateExecutor{ - MetricIdGood: "metric_id_good", - MetricIdTotal: "metric_id_total", - JobId: "job_id", -} - -type mockSlothInfo struct { - window string -} - -// fill in sloth IR with mock information -var requiredSlothInfo = mockSlothInfo{ - window: "1h", -} - -func validatePromQl(_ template.Template) error { - // var intermediate bytes.Buffer - // queryTpl.Execute(&intermediate, mockTemplateExecutor) - // // slothTmpl := template.Must(template.New("").Parse(intermediate.String())) - // // var q bytes.Buffer - // // slothTmpl.Execute(&q, requiredSlothInfo) - // q := strings.ReplaceAll(intermediate.String(), "[{{.window}}]", "") - //_, err := promqlparser.ParseExpr(q) - // if err != nil { - // return err - // } - return nil -} diff --git a/pkg/test/environment.go b/pkg/test/environment.go index 6e36b624f1..5cc8f4faba 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -35,11 +35,8 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" "github.com/onsi/ginkgo/v2" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rancher/opni/pkg/agent" agentv2 "github.com/rancher/opni/pkg/agent/v2" - "github.com/rancher/opni/pkg/alerting/metrics/naming" "github.com/rancher/opni/pkg/alerting/shared" alertingv1 "github.com/rancher/opni/pkg/apis/alerting/v1" corev1 "github.com/rancher/opni/pkg/apis/core/v1" @@ -61,7 +58,6 @@ import ( "github.com/rancher/opni/pkg/plugins" "github.com/rancher/opni/pkg/plugins/hooks" pluginmeta "github.com/rancher/opni/pkg/plugins/meta" - "github.com/rancher/opni/pkg/slo/query" "github.com/rancher/opni/pkg/test/freeport" mock_ident "github.com/rancher/opni/pkg/test/mock/ident" "github.com/rancher/opni/pkg/test/testdata" @@ -1097,141 +1093,6 @@ func (e *Environment) StartOTELCollectorContext(ctx waitctx.PermissiveContext, o return nil } -// Starts a server that exposes Prometheus metrics -// -// Returns port number of the server & a channel that shutdowns the server -func (e *Environment) StartInstrumentationServer() (int, chan struct{}) { - // lg := e.logger - port := freeport.GetFreePort() - - mux := http.NewServeMux() - reg := prometheus.NewRegistry() - - for queryName, queryObj := range query.AvailableQueries { - // register each prometheus collector - reg.MustRegister(queryObj.GetCollector()) - - // create an endpoint simulating good events - mux.HandleFunc(fmt.Sprintf("/%s/%s", queryName, "good"), queryObj.GetGoodEventGenerator()) - // create an endpoint simulating bad events - mux.HandleFunc(fmt.Sprintf("/%s/%s", queryName, "bad"), queryObj.GetBadEventGenerator()) - - } - // expose prometheus metrics - - mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{ - Registry: reg, - })) - - autoInstrumentationServer := &http.Server{ - Addr: fmt.Sprintf("127.0.0.1:%d", port), - Handler: mux, - ReadTimeout: 1 * time.Second, - WriteTimeout: 1 * time.Second, - MaxHeaderBytes: 1 << 20, - } - done := make(chan struct{}) - waitctx.Restrictive.Go(e.ctx, func() { - go func() { - err := autoInstrumentationServer.ListenAndServe() - if !errors.Is(err, http.ErrServerClosed) { - panic(err) - } - }() - defer autoInstrumentationServer.Shutdown(context.Background()) - select { - case <-e.ctx.Done(): - case <-done: - } - }) - return port, done -} - -func (e *Environment) StartMockKubernetesMetricServer() (port int) { - port = freeport.GetFreePort() - - mux := http.NewServeMux() - reg := prometheus.NewRegistry() - - registeredCollectors := map[string]*prometheus.GaugeVec{} - - kubeMetricsIsDefined := prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: naming.KubeMetricsIsDefinedMetricName, - }, []string{"namespace"}) - - kubeMetricsIsDefined.WithLabelValues("kube-system").Set(1) - - setObjHandler := func(w http.ResponseWriter, r *http.Request) { - collectorWriteSync.Lock() - defer collectorWriteSync.Unlock() - objType := r.URL.Query().Get("obj") - name := r.URL.Query().Get("name") - namespace := r.URL.Query().Get("namespace") - phase := r.URL.Query().Get("phase") - uid := r.URL.Query().Get("uid") - - accessedState := false - var b bytes.Buffer - err := naming.KubeObjMetricCreator.Execute(&b, map[string]string{ - "ObjType": objType, - }) - if err != nil { - panic(err) - } - var newOrExistingKubeObjStateCollector *prometheus.GaugeVec - if _, ok := registeredCollectors[b.String()]; !ok { - newOrExistingKubeObjStateCollector = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: b.String(), - }, []string{objType, "namespace", "phase", "uid"}) - registeredCollectors[b.String()] = newOrExistingKubeObjStateCollector - reg.MustRegister(newOrExistingKubeObjStateCollector) - } else { - newOrExistingKubeObjStateCollector = registeredCollectors[b.String()] - } - - for _, validPhase := range shared.KubeStates { - if phase == validPhase { - accessedState = true - newOrExistingKubeObjStateCollector.WithLabelValues(name, namespace, validPhase, uid).Set(1) - } else { - newOrExistingKubeObjStateCollector.WithLabelValues(name, namespace, validPhase, uid).Set(0) - } - } - if accessedState == false { - panic(fmt.Sprintf("Set state for kube metrics api must be one of %s", strings.Join(shared.KubeStates, ","))) - } - } - - reg.MustRegister(kubeMetricsIsDefined) - - mux.HandleFunc("/set", setObjHandler) - mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{ - Registry: reg, - })) - - autoKubernetesMetricsServer := &http.Server{ - Addr: fmt.Sprintf("127.0.0.1:%d", port), - Handler: mux, - ReadTimeout: 1 * time.Second, - WriteTimeout: 1 * time.Second, - MaxHeaderBytes: 1 << 20, - } - waitctx.Permissive.Go(e.ctx, func() { - go func() { - err := autoKubernetesMetricsServer.ListenAndServe() - if err != http.ErrServerClosed { - panic(err) - } - }() - defer autoKubernetesMetricsServer.Shutdown(context.Background()) - select { - case <-e.ctx.Done(): - } - }) - - return port -} - func (e *Environment) StartAgentDisconnectServer() { setDisconnect := func(w http.ResponseWriter, r *http.Request) { agentListMu.Lock() diff --git a/test/plugins/metrics/cortex_query_test.go b/test/plugins/metrics/cortex_query_test.go index f5b71aa3b2..56176adb04 100644 --- a/test/plugins/metrics/cortex_query_test.go +++ b/test/plugins/metrics/cortex_query_test.go @@ -119,5 +119,7 @@ var _ = Describe("Cortex query tests", Ordered, Label("integration"), func() { body, err := io.ReadAll(resp.Body) Expect(err).NotTo(HaveOccurred()) Expect(body).NotTo(BeEmpty()) + + // TODO : MAYBE add other cortex APIS that query metric data in special ways }) }) diff --git a/test/plugins/metrics/cortexadmin_test.go b/test/plugins/metrics/cortexadmin_test.go deleted file mode 100644 index 6b3b5d767f..0000000000 --- a/test/plugins/metrics/cortexadmin_test.go +++ /dev/null @@ -1,490 +0,0 @@ -package metrics_test - -import ( - "context" - "fmt" - "io/ioutil" - "net/http" - "net/url" - "time" - - "github.com/rancher/opni/pkg/alerting/metrics/naming" - "github.com/rancher/opni/pkg/capabilities/wellknown" - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - "github.com/rancher/opni/plugins/metrics/apis/cortexops" - "github.com/tidwall/gjson" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - capabilityv1 "github.com/rancher/opni/pkg/apis/capability/v1" - corev1 "github.com/rancher/opni/pkg/apis/core/v1" - managementv1 "github.com/rancher/opni/pkg/apis/management/v1" - storagev1 "github.com/rancher/opni/pkg/apis/storage/v1" - "github.com/rancher/opni/pkg/test" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "google.golang.org/protobuf/types/known/durationpb" - "google.golang.org/protobuf/types/known/emptypb" -) - -type TestSeriesMetrics struct { - input *cortexadmin.SeriesRequest - output *cortexadmin.SeriesInfoList -} - -type TestMetricLabelSet struct { - input *cortexadmin.LabelRequest - output *cortexadmin.MetricLabels -} - -func expectRuleGroupToExist(ctx context.Context, adminClient cortexadmin.CortexAdminClient, tenant string, groupName string, expectedYaml []byte) error { - for i := 0; i < 10; i++ { - resp, err := adminClient.GetRule(ctx, &cortexadmin.GetRuleRequest{ - ClusterId: tenant, - Namespace: "test", - GroupName: groupName, - }) - if err == nil { - Expect(resp.Data).To(Not(BeNil())) - Expect(resp.Data).To(MatchYAML(expectedYaml)) - return nil - } - time.Sleep(1) - } - return fmt.Errorf("Rule %s should exist, but doesn't", groupName) -} - -func expectRuleGroupToNotExist(ctx context.Context, adminClient cortexadmin.CortexAdminClient, tenant string, groupName string) error { - for i := 0; i < 10; i++ { - _, err := adminClient.GetRule(ctx, &cortexadmin.GetRuleRequest{ - ClusterId: tenant, - Namespace: "test", - GroupName: groupName, - }) - if err != nil { - Expect(status.Code(err)).To(Equal(codes.NotFound)) - return nil - } - - time.Sleep(1) - } - return fmt.Errorf("Rule %s still exists, but shouldn't", groupName) -} - -type mockPod struct { - podName string - namespace string - phase string - uid string -} - -func setMockKubernetesPodState(kubePort int, pod *mockPod) { - queryUrl := fmt.Sprintf("http://localhost:%d/set", kubePort) - client := &http.Client{ - Transport: &http.Transport{}, - } - req, err := http.NewRequest("GET", queryUrl, nil) - if err != nil { - panic(err) - } - values := url.Values{} - values.Set("obj", "pod") - values.Set("name", pod.podName) - values.Set("namespace", pod.namespace) - values.Set("phase", pod.phase) - values.Set("uid", pod.uid) - req.URL.RawQuery = values.Encode() - go func() { - resp, err := client.Do(req) - if err != nil { - panic(err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - panic(fmt.Sprintf("kube metrics prometheus collector hit an error %d", resp.StatusCode)) - } - }() -} - -var _ = XDescribe("Converting ServiceLevelObjective Messages to Prometheus Rules", Ordered, Label("integration", "slow"), func() { - ctx := context.Background() - var env *test.Environment - var adminClient cortexadmin.CortexAdminClient - var kubernetesTempMetricServerPort int - var kubernetesJobName string - ruleTestDataDir := "../../../pkg/test/testdata/testdata/slo/cortexrule" - BeforeAll(func() { - env = &test.Environment{} - Expect(env.Start()).To(Succeed()) - DeferCleanup(env.Stop) - - opsClient := cortexops.NewCortexOpsClient(env.ManagementClientConn()) - _, err := opsClient.ConfigureCluster(context.Background(), &cortexops.ClusterConfiguration{ - Mode: cortexops.DeploymentMode_AllInOne, - Storage: &storagev1.StorageSpec{ - Backend: storagev1.Filesystem, - }, - }) - Expect(err).NotTo(HaveOccurred()) - - client := env.NewManagementClient() - token, err := client.CreateBootstrapToken(context.Background(), &managementv1.CreateBootstrapTokenRequest{ - Ttl: durationpb.New(time.Hour), - }) - Expect(err).NotTo(HaveOccurred()) - info, err := client.CertsInfo(context.Background(), &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - adminClient = cortexadmin.NewCortexAdminClient(env.ManagementClientConn()) - // wait until data has been stored in cortex for the cluster - kubernetesTempMetricServerPort = env.StartMockKubernetesMetricServer() - fmt.Printf("Mock kubernetes metrics server started on port %d\n", kubernetesTempMetricServerPort) - _, errc := env.StartAgent("agent", token, []string{info.Chain[len(info.Chain)-1].Fingerprint}) - Eventually(errc).Should(Receive(BeNil())) - kubernetesJobName = "kubernetes" - env.SetPrometheusNodeConfigOverride("agent", test.NewOverridePrometheusConfig( - "alerting/prometheus/config.yaml", - []test.PrometheusJob{ - { - JobName: kubernetesJobName, - ScrapePort: kubernetesTempMetricServerPort, - }, - }), - ) - _, errc2 := env.StartAgent("agent2", token, []string{info.Chain[len(info.Chain)-1].Fingerprint}) - Eventually(errc2).Should(Receive(BeNil())) - - _, err = client.InstallCapability(context.Background(), &managementv1.CapabilityInstallRequest{ - Name: wellknown.CapabilityMetrics, - Target: &capabilityv1.InstallRequest{ - Cluster: &corev1.Reference{Id: "agent"}, - }, - }) - Expect(err).NotTo(HaveOccurred()) - - _, err = client.InstallCapability(context.Background(), &managementv1.CapabilityInstallRequest{ - Name: wellknown.CapabilityMetrics, - Target: &capabilityv1.InstallRequest{ - Cluster: &corev1.Reference{Id: "agent2"}, - }, - }) - Expect(err).NotTo(HaveOccurred()) - - Eventually(func() error { - stats, err := adminClient.AllUserStats(context.Background(), &emptypb.Empty{}) - if err != nil { - return err - } - for _, item := range stats.Items { - if item.UserID == "agent" { - if item.NumSeries > 0 { - return nil - } - } - } - return fmt.Errorf("waiting for metric data to be stored in cortex") - }, 60*time.Second, 1*time.Second).Should(Succeed()) - - Eventually(func() error { - stats, err := adminClient.AllUserStats(context.Background(), &emptypb.Empty{}) - if err != nil { - return err - } - for _, item := range stats.Items { - if item.UserID == "agent2" { - if item.NumSeries > 0 { - return nil - } - } - } - return fmt.Errorf("waiting for metric data to be stored in cortex") - }, 60*time.Second, 1*time.Second).Should(Succeed()) - - //scrape interval is 1 second - }) - - When("We use the cortex admin plugin", func() { - It("Should be able to enumerate all kube metrics series that match a certain expression", func() { - setMockKubernetesPodState(kubernetesTempMetricServerPort, &mockPod{ - podName: "test-pod", - namespace: "test-namespace", - phase: "Running", - uid: "test-uid", - }) - Eventually(func() error { - resp, err := adminClient.ExtractRawSeries(ctx, &cortexadmin.MatcherRequest{ - Tenant: "agent", - MatchExpr: naming.KubeObjMetricNameMatcher, - }, - ) - if err != nil { - return err - } - - result := gjson.Get(string(resp.Data), "data.result") - if !result.Exists() { - return fmt.Errorf("no result data") - } - if len(result.Array()) == 0 { - return fmt.Errorf("no results") - } - return nil - }, 3*time.Minute, 30*time.Second).Should(Succeed()) - - }) - - It("Should be able to list distinct metrics for each job ", func() { - // expected outputs are a subset of the actual outputs - inputs := []TestSeriesMetrics{ - { - input: &cortexadmin.SeriesRequest{ - Tenant: "agent", - JobId: "prometheus", - }, - output: &cortexadmin.SeriesInfoList{ - Items: []*cortexadmin.SeriesInfo{ - { - SeriesName: "up", - }, - { - SeriesName: "prometheus_http_requests_total", - }, - }, - }, - }, - { - input: &cortexadmin.SeriesRequest{ - Tenant: "agent2", - JobId: "prometheus", - }, - output: &cortexadmin.SeriesInfoList{ - Items: []*cortexadmin.SeriesInfo{ - { - SeriesName: "up", - }, - { - SeriesName: "prometheus_http_requests_total", - }, - }, - }, - }, - } - for _, input := range inputs { - resp, err := adminClient.GetSeriesMetrics(ctx, input.input) - Expect(err).NotTo(HaveOccurred()) - for _, expected := range input.output.Items { - found := false - for _, item := range resp.Items { - if item.SeriesName == expected.SeriesName { - //FIXME: when the metadata API is working also check metadata here - found = true - break - } - } - Expect(found).To(BeTrue()) - } - } - }) - - It("should be able to fetch metric label pairs for each metric", func() { - // expected outputs are a subset of the actual outputs - inputs := []TestMetricLabelSet{ - { - input: &cortexadmin.LabelRequest{ - Tenant: "agent", - JobId: "prometheus", - MetricName: "prometheus_http_requests_total", - }, - output: &cortexadmin.MetricLabels{ - Items: []*cortexadmin.LabelSet{ - { - Name: "code", - Items: []string{ - "200", - }, - }, - { - Name: "handler", - Items: []string{ - "/-/ready", - "/metrics", - }, - }, - }, - }, - }, - { - input: &cortexadmin.LabelRequest{ - Tenant: "agent2", - JobId: "prometheus", - MetricName: "prometheus_http_requests_total", - }, - output: &cortexadmin.MetricLabels{ - Items: []*cortexadmin.LabelSet{ - { - Name: "code", - Items: []string{ - "200", - }, - }, - { - Name: "handler", - Items: []string{ - "/-/ready", - }, - }, - }, - }, - }, - } - for _, input := range inputs { - resp, err := adminClient.GetMetricLabelSets(ctx, input.input) - Expect(err).NotTo(HaveOccurred()) - Expect(resp).NotTo(BeNil()) - for _, expected := range input.output.Items { - found := false - for _, item := range resp.Items { - if item.Name == expected.Name { - found = true - for _, expectedItem := range expected.Items { - foundItem := false - for _, item := range item.Items { - if item == expectedItem { - foundItem = true - break - } - } - Expect(foundItem).To(BeTrue()) - } - break - } - } - Expect(found).To(BeTrue()) - } - } - }) - - It("Should be able to create rules from prometheus yaml", func() { - sampleRule := fmt.Sprintf("%s/sampleRule.yaml", ruleTestDataDir) - sampleRuleYamlString, err := ioutil.ReadFile(sampleRule) - Expect(err).To(Succeed()) - _, err = adminClient.LoadRules(ctx, - &cortexadmin.LoadRuleRequest{ - ClusterId: "agent", - Namespace: "test", - YamlContent: sampleRuleYamlString, - }) - Expect(err).To(Succeed()) - - // Note that sloth by default groups its output into a list of rulefmt.RuleGroup called "groups:" - // While we require the list of rulefmt.RuleGroup to be separated by "---\n" - slothGeneratedGroup := fmt.Sprintf("%s/slothGeneratedGroup.yaml", ruleTestDataDir) - slothGeneratedGroupYamlString, err := ioutil.ReadFile(slothGeneratedGroup) - Expect(err).To(Succeed()) - _, err = adminClient.LoadRules(ctx, - &cortexadmin.LoadRuleRequest{ - ClusterId: "agent", - Namespace: "test", - YamlContent: slothGeneratedGroupYamlString, - }) - Expect(err).To(Succeed()) - Expect(err).NotTo(HaveOccurred()) - - Eventually(func() error { - return expectRuleGroupToExist( - ctx, adminClient, "agent", - "opni-test-slo-rule", sampleRuleYamlString) - }).Should(Succeed()) - - }) - - It("Should be able to update existing rule groups", func() { - sampleRuleUpdate := fmt.Sprintf("%s/sampleRuleUpdate.yaml", ruleTestDataDir) - sampleRuleYamlUpdateString, err := ioutil.ReadFile(sampleRuleUpdate) - Expect(err).To(Succeed()) - _, err = adminClient.LoadRules(ctx, - &cortexadmin.LoadRuleRequest{ - ClusterId: "agent", - Namespace: "test", - YamlContent: sampleRuleYamlUpdateString, - }) - Expect(err).To(Succeed()) - - Eventually(func() error { - return expectRuleGroupToExist( - ctx, adminClient, "agent", - "opni-test-slo-rule", sampleRuleYamlUpdateString) - }).Should(Succeed()) - }) - - It("Should be able to delete existing rule groups", func() { - deleteGroupName := "opni-test-slo-rule" - _, err := adminClient.DeleteRule(ctx, &cortexadmin.DeleteRuleRequest{ - ClusterId: "agent", - Namespace: "test", - GroupName: deleteGroupName, - }) - Expect(err).To(Succeed()) - - // Should find no rule named "opni-test-slo-rule" after deletion - Eventually(func() error { - return expectRuleGroupToNotExist( - ctx, adminClient, "agent", - "opni-test-slo-rule") - }).Should(Succeed()) - }) - }) - When("We are in a multitenant environment", func() { - It("Should be able to apply rules across tenants", func() { - sampleRule := fmt.Sprintf("%s/sampleRule.yaml", ruleTestDataDir) - sampleRuleYamlString, err := ioutil.ReadFile(sampleRule) - Expect(err).To(Succeed()) - _, err = adminClient.LoadRules(ctx, - &cortexadmin.LoadRuleRequest{ - ClusterId: "agent", - Namespace: "test", - YamlContent: sampleRuleYamlString, - }) - Expect(err).To(Succeed()) - _, err = adminClient.LoadRules(ctx, - &cortexadmin.LoadRuleRequest{ - Namespace: "test", - ClusterId: "agent2", - YamlContent: sampleRuleYamlString, - }) - Expect(err).To(Succeed()) - - Eventually(func() error { - return expectRuleGroupToExist( - ctx, adminClient, - "agent", "opni-test-slo-rule", sampleRuleYamlString) - }).Should(Succeed()) - - Eventually(func() error { - return expectRuleGroupToExist( - ctx, adminClient, - "agent2", "opni-test-slo-rule", sampleRuleYamlString, - ) - }).Should(Succeed()) - - deleteGroupName := "opni-test-slo-rule" - _, err = adminClient.DeleteRule(ctx, &cortexadmin.DeleteRuleRequest{ - ClusterId: "agent", - Namespace: "test", - GroupName: deleteGroupName, - }) - Expect(err).To(Succeed()) - - Eventually(func() error { - return expectRuleGroupToExist( - ctx, adminClient, "agent2", - "opni-test-slo-rule", sampleRuleYamlString) - }).Should(Succeed()) - - Eventually(func() error { - return expectRuleGroupToNotExist( - ctx, adminClient, "agent", - "opni-test-slo-rule") - }).Should(Succeed()) - }) - }) -}) From c53598e622becf03e2e0007014cabb191f0bef55 Mon Sep 17 00:00:00 2001 From: Alexandre Lamarre Date: Mon, 18 Sep 2023 17:21:45 -0400 Subject: [PATCH 6/7] mwmb improvements --- pkg/slo/backend/metrics/backend.go | 30 +++- pkg/slo/backend/metrics/backend_test.go | 66 ++++++++- pkg/slo/backend/metrics/manager.go | 89 ++++++++--- pkg/slo/backend/metrics/manager_test.go | 162 ++++++++++++++++++++- pkg/slo/backend/metrics/prometheus.go | 150 ++++++++++++------- pkg/slo/backend/metrics/prometheus_test.go | 61 ++++---- pkg/slo/backend/metrics/window.go | 81 +++++++++-- pkg/slo/backend/metrics/window_test.go | 45 ++++-- test/plugins/metrics/slo_test.go | 6 +- 9 files changed, 545 insertions(+), 145 deletions(-) diff --git a/pkg/slo/backend/metrics/backend.go b/pkg/slo/backend/metrics/backend.go index 5a3c06ad37..3cb176efc2 100644 --- a/pkg/slo/backend/metrics/backend.go +++ b/pkg/slo/backend/metrics/backend.go @@ -5,6 +5,7 @@ import ( "fmt" slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/rancher/opni/pkg/metrics/compat" "github.com/rancher/opni/pkg/slo/backend" "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" "github.com/tidwall/gjson" @@ -24,8 +25,8 @@ var _ backend.ServiceBackend = &MetricsBackend{} func (m *MetricsBackend) ListServices(ctx context.Context, req *slov1.ListServicesRequest) (*slov1.ServiceList, error) { services := &slov1.ServiceList{} + // we should move this to a more generic label discovery API discoveryQuery := `group by (job) ({__name__!=""})` - // TODO : this should probably be a metrics API resp, err := m.adminClient.Get().Query( ctx, &cortexadmin.QueryRequest{ @@ -35,9 +36,34 @@ func (m *MetricsBackend) ListServices(ctx context.Context, req *slov1.ListServic if err != nil { return nil, err } + + qr, err := compat.UnmarshalPrometheusResponse(resp.Data) + if err != nil { + return nil, err + } + + vec, err := qr.GetVector() + if err != nil { + return nil, err + } + if vec == nil { + return nil, fmt.Errorf("could not unmarshal cortex query response to json : expeted model.Vector") + } + for _, vecSample := range *vec { + metric := vecSample.Metric + job, ok := metric["job"] + if !ok { + continue + } + services.Items = append(services.Items, &slov1.Service{ + ClusterId: req.GetClusterId(), + ServiceId: string(job), + }) + } + result := gjson.Get(string(resp.Data), "data.result.#.metric.job") if !result.Exists() { - return nil, fmt.Errorf("could not convert prometheus service discovery to json") + return nil, fmt.Errorf("could not unmarshal cortex query response to expected json") } for _, v := range result.Array() { services.Items = append(services.Items, &slov1.Service{ diff --git a/pkg/slo/backend/metrics/backend_test.go b/pkg/slo/backend/metrics/backend_test.go index b7362a6983..e1f18e5471 100644 --- a/pkg/slo/backend/metrics/backend_test.go +++ b/pkg/slo/backend/metrics/backend_test.go @@ -2,15 +2,20 @@ package metrics_test import ( "context" + "encoding/json" "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/prometheus/common/model" slov1 "github.com/rancher/opni/pkg/apis/slo/v1" "github.com/rancher/opni/pkg/logger" + "github.com/rancher/opni/pkg/metrics/compat" "github.com/rancher/opni/pkg/slo/backend/metrics" cortexadmin_mock "github.com/rancher/opni/pkg/test/mock/cortexadmin" + "github.com/rancher/opni/pkg/test/testutil" "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" + "google.golang.org/grpc" ) var _ = Describe("Metrics SLO Service backend", Label("unit"), Ordered, func() { @@ -18,6 +23,50 @@ var _ = Describe("Metrics SLO Service backend", Label("unit"), Ordered, func() { BeforeAll(func() { ctrl := gomock.NewController(GinkgoT()) mockClient := cortexadmin_mock.NewMockCortexAdminClient(ctrl) + mockClient.EXPECT(). + Query(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, req *cortexadmin.QueryRequest, _ ...grpc.CallOption) (*cortexadmin.QueryResponse, error) { + v := model.Vector{ + { + Metric: model.Metric{ + "job": "test-service", + }, + Value: 1, + Timestamp: model.Time(100000), + }, + } + + wrapper := struct { + Type model.ValueType `json:"resultType"` + Result json.RawMessage `json:"result"` + }{} + wrapper.Type = model.ValVector + wrapperBytes, err := json.Marshal(v) + if err != nil { + return nil, nil + } + wrapper.Result = wrapperBytes + + vectorData, err := json.Marshal(wrapper) + if err != nil { + return nil, err + } + + promResp := compat.ApiResponse{ + Status: "success", + Data: vectorData, + ErrorType: "", + Error: "", + Warnings: []string{}, + } + data, err := json.Marshal(promResp) + if err != nil { + return nil, err + } + return &cortexadmin.QueryResponse{ + Data: data, + }, nil + }).AnyTimes() mockClient.EXPECT(). GetSeriesMetrics(gomock.Any(), gomock.Any()). @@ -45,7 +94,7 @@ var _ = Describe("Metrics SLO Service backend", Label("unit"), Ordered, func() { mb = metrics.NewBackend(m) }) When("we we use the metrics service backend", func() { - XIt("should list discoverable services", func() { //FIXME: TODO: + It("should list discoverable services", func() { svcList, err := mb.ListServices( context.TODO(), &slov1.ListServicesRequest{ @@ -55,11 +104,11 @@ var _ = Describe("Metrics SLO Service backend", Label("unit"), Ordered, func() { Expect(err).NotTo(HaveOccurred()) Expect(svcList.Items).To(ContainElement(&slov1.Service{ ClusterId: "test", - ServiceId: "test", + ServiceId: "test-service", })) }) - XIt("should list metrics based on services", func() { + It("should list metrics based on services", func() { metricList, err := mb.ListMetrics( context.TODO(), &slov1.ListMetricsRequest{ @@ -69,9 +118,14 @@ var _ = Describe("Metrics SLO Service backend", Label("unit"), Ordered, func() { }, ) Expect(err).NotTo(HaveOccurred()) - Expect(metricList).To(Equal(&slov1.MetricGroupList{ - GroupNameToMetrics: map[string]*slov1.MetricList{}, - })) + expected := &slov1.MetricGroupList{ + GroupNameToMetrics: map[string]*slov1.MetricList{ + "other metrics": { + Items: []*slov1.Metric{{Id: "test", Metadata: &slov1.MetricMetadata{}}}, + }, + }, + } + Expect(metricList).To(testutil.ProtoEqual(expected)) }) It("should list events based on metrics and services", func() { diff --git a/pkg/slo/backend/metrics/manager.go b/pkg/slo/backend/metrics/manager.go index 0db66ff8ad..f614549628 100644 --- a/pkg/slo/backend/metrics/manager.go +++ b/pkg/slo/backend/metrics/manager.go @@ -217,7 +217,10 @@ func (m *MetricsSLOStore) Status(ctx context.Context, slo *slov1.SLOData) (*slov if err != nil { return nil, err } - sliExpr := sloGen.SLI(dur).Expr() + sliExpr := sloGen.SLI(WindowMetadata{ + WindowDur: dur, + Name: "test", + }).Expr() rawSLIRes, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ Tenants: []string{clusterId}, Query: sliExpr, @@ -325,7 +328,7 @@ func (m *MetricsSLOStore) Preview(ctx context.Context, slo *slov1.CreateSLOReque if err != nil { return nil, err } - sli := sloGen.SLI(period).Expr() + sli := sloGen.SLI(WindowMetadata{WindowDur: period, Name: "test"}).Expr() cur := time.Now() startTs, endTs := cur.Add(time.Duration(-period)), cur numSteps := 250 @@ -363,28 +366,70 @@ func (m *MetricsSLOStore) Preview(ctx context.Context, slo *slov1.CreateSLOReque } } - // TODO: detect alert windows - // pageAlert := sloGen.PageAlert().Expr() - // rawPageAlert, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ - // Tenants: []string{slo.Slo.ClusterId}, - // Query: pageAlert, - // }) - // fmt.Println(rawPageAlert) - // if err != nil { - // return nil, err - // } - // // TODO : check values - // ticketAlert := sloGen.TicketAlert().Expr() - // rawTicketAlert, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ - // Tenants: []string{slo.Slo.ClusterId}, - // Query: ticketAlert, - // }) - // fmt.Println(rawTicketAlert) - // if err != nil { - // return nil, err - // } + pageIntervals := sloGen.PageIntervals().Expr() + rawPageAlert, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ + Tenants: []string{slo.Slo.ClusterId}, + Query: pageIntervals, + }) + if err != nil { + return nil, err + } + + qrPage, err := compat.UnmarshalPrometheusResponse(rawPageAlert.Data) + if err != nil { + return nil, err + } + + pageSamples := qrPage.LinearSamples() + plotVector.Windows = append(plotVector.Windows, DetectFiringIntervals(severeAlertWindow, pageSamples)...) + + ticketIntervals := sloGen.TicketIntervals().Expr() + rawTicketAlert, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ + Tenants: []string{slo.Slo.ClusterId}, + Query: ticketIntervals, + }) + if err != nil { + return nil, err + } + + qrTicket, err := compat.UnmarshalPrometheusResponse(rawTicketAlert.Data) + if err != nil { + return nil, err + } + + ticketSamples := qrTicket.LinearSamples() + plotVector.Windows = append(plotVector.Windows, DetectFiringIntervals(criticalAlertWindow, ticketSamples)...) return &slov1.SLOPreviewResponse{ PlotVector: plotVector, }, nil } + +const severeAlertWindow = "severe" +const criticalAlertWindow = "critical" + +func DetectFiringIntervals(sev string, samples []compat.Sample) []*slov1.AlertFiringWindows { + windows := make([]*slov1.AlertFiringWindows, 0) + open := false + for _, sample := range samples { + if !open && sample.Value > 0 { + // Convert Unix millisecond timestamp to time.Time + t := time.Unix(sample.Timestamp/1000, (sample.Timestamp%1000)*int64(time.Millisecond)) + windows = append(windows, &slov1.AlertFiringWindows{ + Severity: sev, + Start: timestamppb.New(t), + }) + open = true + } else if open && sample.Value == 0 { + t := time.Unix(sample.Timestamp/1000, (sample.Timestamp%1000)*int64(time.Millisecond)) + windows[len(windows)-1].End = timestamppb.New(t) + open = false + } + } + + if open && windows[len(windows)-1].End == nil { + windows[len(windows)-1].End = timestamppb.Now() + } + + return windows +} diff --git a/pkg/slo/backend/metrics/manager_test.go b/pkg/slo/backend/metrics/manager_test.go index 37b71b3664..7379cfeac0 100644 --- a/pkg/slo/backend/metrics/manager_test.go +++ b/pkg/slo/backend/metrics/manager_test.go @@ -2,6 +2,7 @@ package metrics_test import ( "context" + "encoding/json" "errors" "time" @@ -9,20 +10,24 @@ import ( "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/prometheus/common/model" corev1 "github.com/rancher/opni/pkg/apis/core/v1" slov1 "github.com/rancher/opni/pkg/apis/slo/v1" "github.com/rancher/opni/pkg/logger" + "github.com/rancher/opni/pkg/metrics/compat" "github.com/rancher/opni/pkg/slo/backend/metrics" cortexadmin_mock "github.com/rancher/opni/pkg/test/mock/cortexadmin" + "github.com/rancher/opni/pkg/test/testutil" "github.com/rancher/opni/pkg/util" "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/timestamppb" ) -// TODO : maybe move this to a conformance test var _ = Describe("SLO Metrics store", Ordered, Label("unit"), func() { var metricsStore *metrics.MetricsSLOStore BeforeAll(func() { @@ -36,9 +41,95 @@ var _ = Describe("SLO Metrics store", Ordered, Label("unit"), func() { mockClient.EXPECT().GetRule(gomock.Any(), gomock.Any()).Return(nil, status.Error(codes.NotFound, "rule not found")).AnyTimes() - mockClient.EXPECT().Query(gomock.Any(), gomock.Any()).Return(&cortexadmin.QueryResponse{ - Data: []byte{}, - }, nil).AnyTimes() + mockClient.EXPECT().Query(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, req *cortexadmin.QueryRequest, _ ...grpc.CallOption) (*cortexadmin.QueryResponse, error) { + v := model.Vector{ + { + Metric: model.Metric{ + "job": "test-service", + }, + Value: 1, + Timestamp: model.Time(100000), + }, + } + + wrapper := struct { + Type model.ValueType `json:"resultType"` + Result json.RawMessage `json:"result"` + }{} + wrapper.Type = model.ValVector + wrapperBytes, err := json.Marshal(v) + if err != nil { + return nil, nil + } + wrapper.Result = wrapperBytes + + vectorData, err := json.Marshal(wrapper) + if err != nil { + return nil, err + } + + promResp := compat.ApiResponse{ + Status: "success", + Data: vectorData, + ErrorType: "", + Error: "", + Warnings: []string{}, + } + data, err := json.Marshal(promResp) + if err != nil { + return nil, err + } + return &cortexadmin.QueryResponse{ + Data: data, + }, nil + }, + ).AnyTimes() + + mockClient.EXPECT().QueryRange(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, req *cortexadmin.QueryRangeRequest, _ ...grpc.CallOption) (*cortexadmin.QueryResponse, error) { + v := model.Vector{ + { + Metric: model.Metric{ + "job": "test-service", + }, + Value: 1, + Timestamp: model.Time(100000), + }, + } + + wrapper := struct { + Type model.ValueType `json:"resultType"` + Result json.RawMessage `json:"result"` + }{} + wrapper.Type = model.ValVector + wrapperBytes, err := json.Marshal(v) + if err != nil { + return nil, nil + } + wrapper.Result = wrapperBytes + + vectorData, err := json.Marshal(wrapper) + if err != nil { + return nil, err + } + + promResp := compat.ApiResponse{ + Status: "success", + Data: vectorData, + ErrorType: "", + Error: "", + Warnings: []string{}, + } + data, err := json.Marshal(promResp) + if err != nil { + return nil, err + } + return &cortexadmin.QueryResponse{ + Data: data, + }, nil + }, + ).AnyTimes() lg := logger.NewPluginLogger().Named("slo") provider := metrics.NewMetricsSLOStoreProvider(lg) @@ -48,6 +139,63 @@ var _ = Describe("SLO Metrics store", Ordered, Label("unit"), func() { ) }) + It("should detect active SLO windows from samples", func() { + samples := []compat.Sample{ + { + Value: 0, + Timestamp: 1, + }, + { + Value: 5, + Timestamp: 2, + }, + { + Value: 10, + Timestamp: 10, + }, + { + Value: 0, + Timestamp: 11, + }, + { + Value: 0, + Timestamp: 15, + }, + { + Value: 1, + Timestamp: 33, + }, + { + Value: 0, + Timestamp: 34, + }, + { + Value: 55, + Timestamp: 4000, + }, + } + windows := metrics.DetectFiringIntervals("critical", samples) + + convert := func(a int64) time.Time { + return time.Unix(a/1000, (a%1000)*int64(time.Millisecond)) + } + + // t := time.Unix(sample.Timestamp/1000, (sample.Timestamp%1000)*int64(time.Millisecond)) + first := &slov1.AlertFiringWindows{Start: timestamppb.New(convert(2)), End: timestamppb.New(convert(11)), Severity: "critical"} + second := &slov1.AlertFiringWindows{Start: timestamppb.New(convert(33)), End: timestamppb.New(convert(34)), Severity: "critical"} + third := &slov1.AlertFiringWindows{Start: timestamppb.New(convert(4000)), End: timestamppb.New(convert(4000)), Severity: "critical"} + + Expect(len(windows)).To(Equal(3)) + Expect(windows[0]).To(testutil.ProtoEqual(first)) + Expect(windows[1]).To(testutil.ProtoEqual(second)) + last := windows[2] + Expect(last.Start).To(testutil.ProtoEqual(third.Start)) + Expect(last.Severity).To(Equal(third.Severity)) + Expect(last.End).NotTo(BeNil()) + Expect(last.End.AsTime().After(time.Now().Add(-time.Minute))).To(BeTrue()) + + }) + When("we use the mocked metrics store", func() { It("should be able to create metrics SLOs", func() { ref, err := metricsStore.Create(context.TODO(), &slov1.CreateSLORequest{ @@ -115,8 +263,7 @@ var _ = Describe("SLO Metrics store", Ordered, Label("unit"), func() { Expect(err).To(Succeed()) }) - XIt("should be able to fetch the status of metrics SLOs", func() { - // TODO + It("should be able to fetch the status of metrics SLOs", func() { _, err := metricsStore.Status( context.TODO(), &slov1.SLOData{ @@ -127,8 +274,7 @@ var _ = Describe("SLO Metrics store", Ordered, Label("unit"), func() { Expect(err).To(Succeed()) }) - XIt("should fetch previews of the metrics SLOS", func() { - // TODO + It("should fetch previews of the metrics SLOS", func() { _, err := metricsStore.Preview( context.TODO(), &slov1.CreateSLORequest{ diff --git a/pkg/slo/backend/metrics/prometheus.go b/pkg/slo/backend/metrics/prometheus.go index 8c34aac0ce..58987f38e0 100644 --- a/pkg/slo/backend/metrics/prometheus.go +++ b/pkg/slo/backend/metrics/prometheus.go @@ -3,10 +3,8 @@ package metrics import ( "errors" "fmt" - "text/template" "time" - "github.com/prometheus/common/model" prommodel "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/rulefmt" "github.com/rancher/opni/pkg/slo/backend" @@ -45,8 +43,8 @@ type MetricGenerator interface { } type MWMBSLOGenerator interface { - Windows() []prommodel.Duration - SLI(measurementPeriod prommodel.Duration) MetricGenerator + Windows() []WindowMetadata + SLI(WindowMetadata) MetricGenerator Objective() MetricGenerator ErrorBudget() MetricGenerator @@ -57,11 +55,10 @@ type MWMBSLOGenerator interface { PeriodBurnRate() MetricGenerator ErrorBudgetRemaining() MetricGenerator - // TODO : unimplemented - // TicketWindow() MetricGenerator - // PageWindow() MetricGenerator PageAlert() MetricGenerator TicketAlert() MetricGenerator + PageIntervals() MetricGenerator + TicketIntervals() MetricGenerator } type SLOGeneratorOptions struct { @@ -94,7 +91,7 @@ func NewSLOGenerator(slo backend.SLO, opts ...SLOGeneratorOption) (*SLOGenerator if err := slo.Validate(); err != nil { return nil, err } - dur, err := prommodel.ParseDuration(slo.SloPeriod) + sloPeriod, err := prommodel.ParseDuration(slo.SloPeriod) if err != nil { return nil, err } @@ -109,11 +106,13 @@ func NewSLOGenerator(slo backend.SLO, opts ...SLOGeneratorOption) (*SLOGenerator switch slo.SloPeriod { case "28d", "30d": //fallback to google SRE production presets windows = GenerateMWMBWindows( - prommodel.Duration(time.Minute * 5), + sloPeriod, + prommodel.Duration(time.Minute*5), ) default: // use somewhat hacky normalization windows = GenerateMWMBWindows( - prommodel.Duration(NormalizePeriodToBudgetingInterval(time.Duration(dur))), + sloPeriod, + prommodel.Duration(NormalizePeriodToBudgetingInterval(time.Duration(sloPeriod))), ) } @@ -122,7 +121,7 @@ func NewSLOGenerator(slo backend.SLO, opts ...SLOGeneratorOption) (*SLOGenerator options: options, goodQuery: goodQuery, totalQuery: totalQuery, - period: dur, + period: sloPeriod, windows: windows, }, nil } @@ -169,7 +168,7 @@ func (s *SLOGeneratorImpl) rules() ([]rulefmt.RuleNode, error) { s.ErrorBudgetRemaining().Rule, } - for _, window := range append(s.Windows(), s.period) { + for _, window := range s.Windows() { sli := s.SLI(window) ruleConstructor = append(ruleConstructor, sli.Rule) } @@ -187,16 +186,16 @@ func (s *SLOGeneratorImpl) rules() ([]rulefmt.RuleNode, error) { return rules, errors.Join(errs...) } -func (s *SLOGeneratorImpl) Windows() []prommodel.Duration { - return s.windows.windowRangeDur() +func (s *SLOGeneratorImpl) Windows() []WindowMetadata { + return s.windows.WindowRange() } -func (s *SLOGeneratorImpl) SLI(measurementPeriod prommodel.Duration) MetricGenerator { +func (s *SLOGeneratorImpl) SLI(w WindowMetadata) MetricGenerator { return NewMWMBSLIGenerator( SLOSLI, s.goodQuery, s.totalQuery, - measurementPeriod, + w, s.slo.IdLabels, ) } @@ -236,10 +235,11 @@ func (s *SLOGeneratorImpl) CurrentBurnRate() MetricGenerator { } func (s *SLOGeneratorImpl) PeriodBurnRate() MetricGenerator { + periodWindow := s.Windows()[len(s.Windows())-1] return NewBurnRateGenerator( SLOPeriodBurnRate, s.slo.IdLabels, - s.SLI(s.period), + s.SLI(periodWindow), s.ErrorBudget(), s.options, ) @@ -256,27 +256,53 @@ func (s *SLOGeneratorImpl) ErrorBudgetRemaining() MetricGenerator { func (s *SLOGeneratorImpl) PageAlert() MetricGenerator { return NewMWMBAlertGenerator( - "slo:mwmb_alert:page", s.slo.IdLabels, s.windows.PageQuick, s.windows.PageSlow, - func(window prommodel.Duration) MetricGenerator { + func(window WindowMetadata) MetricGenerator { return s.SLI(window) }, s.options, + false, ) } func (s *SLOGeneratorImpl) TicketAlert() MetricGenerator { return NewMWMBAlertGenerator( - "slo:mwmb_alert:ticket", s.slo.IdLabels, s.windows.TicketQuick, s.windows.TicketSlow, - func(window prommodel.Duration) MetricGenerator { + func(window WindowMetadata) MetricGenerator { return s.SLI(window) }, s.options, + false, + ) +} + +func (s *SLOGeneratorImpl) PageIntervals() MetricGenerator { + return NewMWMBAlertGenerator( + s.slo.IdLabels, + s.windows.PageQuick, + s.windows.PageSlow, + func(window WindowMetadata) MetricGenerator { + return s.SLI(window) + }, + s.options, + true, + ) +} + +func (s *SLOGeneratorImpl) TicketIntervals() MetricGenerator { + return NewMWMBAlertGenerator( + s.slo.IdLabels, + s.windows.TicketQuick, + s.windows.TicketSlow, + func(window WindowMetadata) MetricGenerator { + return s.SLI(window) + }, + s.options, + true, ) } @@ -325,14 +351,14 @@ func (v *ConstantMetricGenerator) Rule() (rulefmt.RuleNode, error) { type SumRateGenerator struct { goodExpr string - window prommodel.Duration + window WindowMetadata } var _ MetricGenerator = (*SumRateGenerator)(nil) func NewSumRateGenerator( goodExpr string, - window prommodel.Duration, + window WindowMetadata, ) *SumRateGenerator { return &SumRateGenerator{ goodExpr: goodExpr, @@ -345,7 +371,7 @@ func (s *SumRateGenerator) Id() string { } func (s *SumRateGenerator) Expr() string { - return fmt.Sprintf("sum(rate(%s[%s]))", s.goodExpr, s.window.String()) + return fmt.Sprintf("sum(rate(%s[%s]))", s.goodExpr, s.window.WindowDur.String()) } func (s *SumRateGenerator) Rule() (rulefmt.RuleNode, error) { @@ -354,7 +380,7 @@ func (s *SumRateGenerator) Rule() (rulefmt.RuleNode, error) { func NewMWMBSLIGenerator( name, goodExpr, totalExpr string, - window prommodel.Duration, + window WindowMetadata, idLabels backend.IdentificationLabels, ) *MWMBSLIGenerator { goodQuery := NewSumRateGenerator(goodExpr, window) @@ -370,7 +396,7 @@ func NewMWMBSLIGenerator( type MWMBSLIGenerator struct { name string - window prommodel.Duration + window WindowMetadata goodQuery MetricGenerator totalQuery MetricGenerator @@ -381,7 +407,7 @@ type MWMBSLIGenerator struct { var _ MetricGenerator = (*MWMBSLIGenerator)(nil) func (s *MWMBSLIGenerator) Id() string { - return s.name + s.window.String() + return s.name + ":" + s.window.Name } func (s *MWMBSLIGenerator) Expr() string { @@ -520,63 +546,85 @@ func (e *ErrorBudgetRemainingGenerator) Rule() (rulefmt.RuleNode, error) { }, nil } -var mwmbAlertTplBool = template.Must(template.New("mwmbAlertTpl").Option("missingkey=error").Parse(` -(max({{ .QuickShortMetric }}{{ .MetricFilter}} > bool ({{ .QuickShortBurnFactor }} * {{ .ErrorBudgetRatio }})) -without ({{ .WindowLabel }}) and -max({{ .QuickLongMetric }}{{ .MetricFilter}} > bool ({{ .QuickLongBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }})) - -or - -(max({{ .SlowShortMetric }}{{ .MetricFilter }} > bool ({{ .SlowShortBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }}) -and max({{ .SlowQuickMetric }}{{ .MetricFilter }} > bool ({{ .SlowQuickBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }}))`)) - // TODO : add a description of how these alerts are structured type MWMBAlertGenerator struct { - name string quickWindow Window slowWindow Window idLabels backend.IdentificationLabels options *SLOGeneratorOptions - errorRate func(model.Duration) MetricGenerator + errorRate func(WindowMetadata) MetricGenerator + asRule bool } var _ MetricGenerator = (*MWMBAlertGenerator)(nil) func NewMWMBAlertGenerator( - name string, idLabels backend.IdentificationLabels, quickWindow, slowWindow Window, - errorRate func(model.Duration) MetricGenerator, + errorRate func(WindowMetadata) MetricGenerator, options *SLOGeneratorOptions, + asRule bool, ) *MWMBAlertGenerator { return &MWMBAlertGenerator{ - name: name, idLabels: idLabels, quickWindow: quickWindow, slowWindow: slowWindow, errorRate: errorRate, options: options, + asRule: asRule, } } func (m *MWMBAlertGenerator) Id() string { - return m.name + baseID := "slo:mwmb_alert" + if m.asRule { + return baseID + ":interval:" + m.quickWindow.Name() + } + return baseID + ":" + m.quickWindow.Name() } func (m *MWMBAlertGenerator) Expr() string { - construct := func(dur prommodel.Duration) string { - return m.errorRate(dur).Expr() + construct := func(w WindowMetadata) string { + return m.errorRate(w).Expr() } if m.options.optimized { - construct = func(dur prommodel.Duration) string { - return fmt.Sprintf("%s{%s}", m.errorRate(dur).Id(), m.idLabels.ToLabels().ConstructPrometheus()) + construct = func(w WindowMetadata) string { + return fmt.Sprintf("%s{%s}", m.errorRate(w).Id(), m.idLabels.ToLabels().ConstructPrometheus()) } } - shortWindowQuickExpr := fmt.Sprintf("(%s) > (%.9f * %.9f)", construct(m.quickWindow.ShortWindow), m.quickWindow.GetShortBurnRateFactor(), m.quickWindow.ErrorBudgetPercent) - shortWindowLongExpr := fmt.Sprintf("(%s) > (%.9f * %.9f)", construct(m.quickWindow.LongWindow), m.quickWindow.GetLongBurnRateFactor(), m.quickWindow.ErrorBudgetPercent) - longWindowQuickExpr := fmt.Sprintf("(%s) > (%.9f * %.9f)", construct(m.slowWindow.ShortWindow), m.slowWindow.GetShortBurnRateFactor(), m.slowWindow.ErrorBudgetPercent) - longWindowLongExpr := fmt.Sprintf("(%s) > (%.9f * %.9f)", construct(m.slowWindow.LongWindow), m.slowWindow.GetLongBurnRateFactor(), m.slowWindow.ErrorBudgetPercent) + compOperator := ">" + if m.asRule { + compOperator = "> bool" + } + shortWindowQuickExpr := fmt.Sprintf( + "(%s) %s (%.9f * %.9f)", + construct(m.quickWindow.ShortWindowMetadata()), + compOperator, + m.quickWindow.GetShortBurnRateFactor(), + m.quickWindow.ErrorBudgetPercent, + ) + shortWindowLongExpr := fmt.Sprintf( + "(%s) %s (%.9f * %.9f)", + construct(m.quickWindow.LongWindowMetadata()), + compOperator, + m.quickWindow.GetLongBurnRateFactor(), + m.quickWindow.ErrorBudgetPercent, + ) + longWindowQuickExpr := fmt.Sprintf( + "(%s) %s (%.9f * %.9f)", + construct(m.slowWindow.ShortWindowMetadata()), + compOperator, + m.slowWindow.GetShortBurnRateFactor(), + m.slowWindow.ErrorBudgetPercent, + ) + longWindowLongExpr := fmt.Sprintf( + "(%s) %s (%.9f * %.9f)", + construct(m.slowWindow.LongWindowMetadata()), + compOperator, + m.slowWindow.GetLongBurnRateFactor(), + m.slowWindow.ErrorBudgetPercent, + ) return fmt.Sprintf( "(max((%s)) and max((%s))) or (max(%s) and max(%s))", diff --git a/pkg/slo/backend/metrics/prometheus_test.go b/pkg/slo/backend/metrics/prometheus_test.go index d60e8251d0..3520b10843 100644 --- a/pkg/slo/backend/metrics/prometheus_test.go +++ b/pkg/slo/backend/metrics/prometheus_test.go @@ -39,7 +39,7 @@ var _ = Describe("Prometheus SLO", Label("unit"), func() { }) Specify("sum rate generators should produce valid promQL", func() { - sr := metrics.NewSumRateGenerator("hello_vector", prommodel.Duration(time.Minute)) + sr := metrics.NewSumRateGenerator("hello_vector", metrics.WindowMetadata{WindowDur: prommodel.Duration(time.Minute)}) // this is a partial generator Expect(sr.Id()).To(Equal("")) @@ -50,7 +50,7 @@ var _ = Describe("Prometheus SLO", Label("unit"), func() { _, err = promql.ParseExpr(sr.Expr()) Expect(err).NotTo(HaveOccurred()) - seconds := metrics.NewSumRateGenerator("hello_vector", prommodel.Duration(time.Second)) + seconds := metrics.NewSumRateGenerator("hello_vector", metrics.WindowMetadata{WindowDur: prommodel.Duration(time.Second)}) Expect(seconds.Id()).To(Equal("")) Expect(seconds.Expr()).To(Equal("sum(rate(hello_vector[1s]))")) @@ -60,14 +60,14 @@ var _ = Describe("Prometheus SLO", Label("unit"), func() { _, err = promql.ParseExpr(seconds.Expr()) Expect(err).NotTo(HaveOccurred()) - hours := metrics.NewSumRateGenerator("hello_vector", prommodel.Duration(time.Hour)) + hours := metrics.NewSumRateGenerator("hello_vector", metrics.WindowMetadata{WindowDur: prommodel.Duration(time.Hour)}) Expect(hours.Id()).To(Equal("")) Expect(hours.Expr()).To(Equal("sum(rate(hello_vector[1h]))")) _, err = hours.Rule() Expect(err).To(HaveOccurred()) - days := metrics.NewSumRateGenerator("hello_vector", prommodel.Duration(24*time.Hour)) + days := metrics.NewSumRateGenerator("hello_vector", metrics.WindowMetadata{WindowDur: prommodel.Duration(24 * time.Hour)}) Expect(days.Id()).To(Equal("")) Expect(days.Expr()).To(Equal("sum(rate(hello_vector[1d]))")) @@ -80,13 +80,13 @@ var _ = Describe("Prometheus SLO", Label("unit"), func() { "sli_vector", "http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}", "http_request_duration_seconds_count{job=\"myservice\"}", - prommodel.Duration(time.Minute), + metrics.WindowMetadata{WindowDur: prommodel.Duration(time.Minute), Name: "page:quick:short"}, map[string]string{ "bar": "baz", }, ) - Expect(mwmb.Id()).To(Equal("sli_vector1m")) + Expect(mwmb.Id()).To(Equal("sli_vector:page:quick:short")) Expect(mwmb.Expr()).To(Equal( "1 - ((sum(rate(http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}[1m]))) / (sum(rate(http_request_duration_seconds_count{job=\"myservice\"}[1m]))))", )) @@ -107,13 +107,13 @@ var _ = Describe("Prometheus SLO", Label("unit"), func() { "sli_vector", "http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}", "http_request_duration_seconds_count{job=\"myservice\"}", - prommodel.Duration(time.Second), + metrics.WindowMetadata{WindowDur: prommodel.Duration(time.Second), Name: "ticket:long:long"}, map[string]string{ "bar": "baz", }, ) - Expect(seconds.Id()).To(Equal("sli_vector1s")) + Expect(seconds.Id()).To(Equal("sli_vector:ticket:long:long")) Expect(seconds.Expr()).To(Equal( "1 - ((sum(rate(http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}[1s]))) / (sum(rate(http_request_duration_seconds_count{job=\"myservice\"}[1s]))))", )) @@ -134,13 +134,13 @@ var _ = Describe("Prometheus SLO", Label("unit"), func() { "sli_vector", "http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}", "http_request_duration_seconds_count{job=\"myservice\"}", - prommodel.Duration(time.Hour*6), + metrics.WindowMetadata{WindowDur: prommodel.Duration(time.Hour * 6), Name: "page:quick:short"}, map[string]string{ "bar": "baz", }, ) - Expect(hours.Id()).To(Equal("sli_vector6h")) + Expect(hours.Id()).To(Equal("sli_vector:page:quick:short")) Expect(hours.Expr()).To(Equal( "1 - ((sum(rate(http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}[6h]))) / (sum(rate(http_request_duration_seconds_count{job=\"myservice\"}[6h]))))", )) @@ -161,13 +161,13 @@ var _ = Describe("Prometheus SLO", Label("unit"), func() { "sli_vector", "http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}", "http_request_duration_seconds_count{job=\"myservice\"}", - prommodel.Duration(time.Hour*24*24), + metrics.WindowMetadata{WindowDur: prommodel.Duration(time.Hour * 24 * 24), Name: "test"}, map[string]string{ "bar": "baz", }, ) - Expect(days.Id()).To(Equal("sli_vector24d")) + Expect(days.Id()).To(Equal("sli_vector:test")) Expect(days.Expr()).To(Equal( "1 - ((sum(rate(http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}[24d]))) / (sum(rate(http_request_duration_seconds_count{job=\"myservice\"}[24d]))))", )) @@ -327,7 +327,7 @@ var _ = Describe("Prometheus SLO", Label("unit"), func() { "id": "1", "foo": "bar", } - errorRate := func(window prommodel.Duration) metrics.MetricGenerator { + errorRate := func(window metrics.WindowMetadata) metrics.MetricGenerator { return metrics.NewMWMBSLIGenerator( "error_rate", "test_metric{foo=~\"bar\",id=~\"1\"}", @@ -340,17 +340,18 @@ var _ = Describe("Prometheus SLO", Label("unit"), func() { mwmb := metrics.WindowDefaults(prommodel.Duration(time.Hour * 24 * 30)) quick, long := mwmb.PageWindows() pageGen := metrics.NewMWMBAlertGenerator( - "mwmb_alert", idLabels, quick, long, errorRate, opts, + false, ) - Expect(pageGen.Id()).To(Equal("mwmb_alert")) + Expect(pageGen.Id()).To(Equal("slo:mwmb_alert:page:quick")) GinkgoWriter.Write([]byte(pageGen.Expr())) - // Expect(pageGen.Expr()).Should(Equal("TODO")) // TODO + expected := "(max(((error_rate:page:quick:short{foo=~\"bar\",id=~\"1\"}) > (172.800000000 * 2.000000000))) and max(((error_rate:page:quick:long{foo=~\"bar\",id=~\"1\"}) > (14.400000000 * 2.000000000)))) or (max((error_rate:page:slow:short{foo=~\"bar\",id=~\"1\"}) > (72.000000000 * 5.000000000)) and max((error_rate:page:slow:long{foo=~\"bar\",id=~\"1\"}) > (6.000000000 * 5.000000000)))" + Expect(pageGen.Expr()).Should(Equal(expected)) rule, err := pageGen.Rule() Expect(err).NotTo(HaveOccurred()) @@ -368,16 +369,17 @@ var _ = Describe("Prometheus SLO", Label("unit"), func() { metrics.WithOptimization(false), ) pageGenRaw := metrics.NewMWMBAlertGenerator( - "mwmb_alert", idLabels, quick, long, errorRate, opts, + false, ) - Expect(pageGenRaw.Id()).To(Equal("mwmb_alert")) - // Expect(pageGenRaw.Expr()).Should(Equal("TODO")) // TODO + Expect(pageGenRaw.Id()).To(Equal("slo:mwmb_alert:page:quick")) + expectedRaw := "(max(((1 - ((sum(rate(test_metric{foo=~\"bar\",id=~\"1\"}[5m]))) / (sum(rate(test_metric{foo=~\"bar\",id=~\"1|2\"}[5m]))))) > (172.800000000 * 2.000000000))) and max(((1 - ((sum(rate(test_metric{foo=~\"bar\",id=~\"1\"}[1h]))) / (sum(rate(test_metric{foo=~\"bar\",id=~\"1|2\"}[1h]))))) > (14.400000000 * 2.000000000)))) or (max((1 - ((sum(rate(test_metric{foo=~\"bar\",id=~\"1\"}[30m]))) / (sum(rate(test_metric{foo=~\"bar\",id=~\"1|2\"}[30m]))))) > (72.000000000 * 5.000000000)) and max((1 - ((sum(rate(test_metric{foo=~\"bar\",id=~\"1\"}[6h]))) / (sum(rate(test_metric{foo=~\"bar\",id=~\"1|2\"}[6h]))))) > (6.000000000 * 5.000000000)))" + Expect(pageGenRaw.Expr()).Should(Equal(expectedRaw)) rule, err = pageGenRaw.Rule() Expect(err).NotTo(HaveOccurred()) @@ -436,9 +438,9 @@ var _ = Describe("Prometheus SLO", Label("unit"), func() { Expect(gen).NotTo(BeNil()) Expect(gen.Id()).NotTo(Equal("")) Expect(gen.Id()).To(HavePrefix(metrics.SLOSLI)) - Expect(gen.Id()).To(HaveSuffix(prommodel.Duration(window).String())) - Expect(gen.Id()).To(Equal(metrics.SLOSLI + prommodel.Duration(window).String())) - expected := fmt.Sprintf("1 - ((sum(rate(test_metric{job=\"scrape\", foo=~\"bar\"}[%s]))) / (sum(rate(test_metric{job=\"scrape\"}[%s]))))", prommodel.Duration(window), prommodel.Duration(window)) + Expect(gen.Id()).To(HaveSuffix(window.Name)) + Expect(gen.Id()).To(Equal(metrics.SLOSLI + ":" + window.Name)) + expected := fmt.Sprintf("1 - ((sum(rate(test_metric{job=\"scrape\", foo=~\"bar\"}[%s]))) / (sum(rate(test_metric{job=\"scrape\"}[%s]))))", prommodel.Duration(window.WindowDur), prommodel.Duration(window.WindowDur)) Expect(gen.Expr()).To(Equal(expected)) _, err = promql.ParseExpr(gen.Expr()) Expect(err).NotTo(HaveOccurred()) @@ -556,7 +558,7 @@ var _ = Describe("Prometheus SLO", Label("unit"), func() { By("expecting the optimized period burn rate rule to reference the correct sli rule") pbr := sloGen.PeriodBurnRate() - periodSLIID := sloGen.SLI(util.Must(prommodel.ParseDuration(period))).Id() + periodSLIID := sloGen.SLI(metrics.WindowMetadata{WindowDur: util.Must(prommodel.ParseDuration(period)), Name: "period"}).Id() expectedP := fmt.Sprintf( `%s{%s} / on(%s) group_left %s{%s}`, @@ -566,6 +568,7 @@ var _ = Describe("Prometheus SLO", Label("unit"), func() { errorBudgetId, curSloFilters, ) + GinkgoWriter.Write([]byte(expectedP)) Expect(pbr.Expr()).To(Equal(expectedP)) _, err = promql.ParseExpr(pbr.Expr()) @@ -658,9 +661,9 @@ var _ = Describe("Prometheus SLO", Label("unit"), func() { Expect(gen).NotTo(BeNil()) Expect(gen.Id()).NotTo(Equal("")) Expect(gen.Id()).To(HavePrefix(metrics.SLOSLI)) - Expect(gen.Id()).To(HaveSuffix(prommodel.Duration(window).String())) - Expect(gen.Id()).To(Equal(metrics.SLOSLI + prommodel.Duration(window).String())) - expected := fmt.Sprintf("1 - ((sum(rate(test_metric{job=\"scrape\", foo=~\"bar\",code=~\"200\"}[%s]))) / (sum(rate(test_metric{job=\"scrape\", code=~\"200|500|503\"}[%s]))))", prommodel.Duration(window), prommodel.Duration(window)) + Expect(gen.Id()).To(HaveSuffix(window.Name)) + Expect(gen.Id()).To(Equal(metrics.SLOSLI + ":" + window.Name)) + expected := fmt.Sprintf("1 - ((sum(rate(test_metric{job=\"scrape\", foo=~\"bar\",code=~\"200\"}[%s]))) / (sum(rate(test_metric{job=\"scrape\", code=~\"200|500|503\"}[%s]))))", prommodel.Duration(window.WindowDur), prommodel.Duration(window.WindowDur)) Expect(gen.Expr()).To(Equal(expected)) _, err := promql.ParseExpr(gen.Expr()) Expect(err).NotTo(HaveOccurred()) @@ -770,7 +773,7 @@ var _ = Describe("Prometheus SLO", Label("unit"), func() { By("expecting the unoptimized period burn rate rule to be valid promql") pbr := sloGen.PeriodBurnRate() - periodSLIExpr := sloGen.SLI(util.Must(prommodel.ParseDuration(period))).Expr() + periodSLIExpr := sloGen.SLI(metrics.WindowMetadata{WindowDur: util.Must(prommodel.ParseDuration(period)), Name: "test"}).Expr() expectedP := fmt.Sprintf( `(%s) / (%s)`, @@ -884,7 +887,7 @@ var _ = Describe("Prometheus SLO", Label("unit"), func() { sloGen.PeriodBurnRate().Id(), sloGen.ErrorBudgetRemaining().Id(), } - for _, window := range append(sloGen.Windows(), period) { + for _, window := range append(sloGen.Windows()) { generatorIds = append(generatorIds, sloGen.SLI(window).Id()) } @@ -969,7 +972,7 @@ var _ = Describe("Prometheus SLO", Label("unit"), func() { sloGen.PeriodBurnRate().Id(), sloGen.ErrorBudgetRemaining().Id(), } - for _, window := range append(sloGen.Windows(), period) { + for _, window := range append(sloGen.Windows()) { generatorIds = append(generatorIds, sloGen.SLI(window).Id()) } diff --git a/pkg/slo/backend/metrics/window.go b/pkg/slo/backend/metrics/window.go index 7d038e0f9b..61f910e275 100644 --- a/pkg/slo/backend/metrics/window.go +++ b/pkg/slo/backend/metrics/window.go @@ -7,7 +7,6 @@ import ( "slices" "github.com/prometheus/common/model" - "github.com/samber/lo" ) func NormalizePeriodToBudgetingInterval(period time.Duration) (budgetingInterval time.Duration) { @@ -17,6 +16,7 @@ func NormalizePeriodToBudgetingInterval(period time.Duration) (budgetingInterval } type MWMBWindows struct { + Period model.Duration PageQuick Window PageSlow Window TicketQuick Window @@ -32,21 +32,25 @@ func (w MWMBWindows) windows() []Window { } } -func (w MWMBWindows) WindowRange() []string { - windows := w.windowRangeDur() - return lo.Map(windows, func(w model.Duration, _ int) string { - return w.String() - }) +type WindowMetadata struct { + WindowDur model.Duration + // use for building other rules + Name string } -func (w MWMBWindows) windowRangeDur() []model.Duration { - windows := []model.Duration{} +func (w MWMBWindows) WindowRange() []WindowMetadata { + windows := []WindowMetadata{} for _, w := range w.windows() { - windows = append(windows, w.ShortWindow) - windows = append(windows, w.LongWindow) + windows = append(windows, w.ShortWindowMetadata()) + windows = append(windows, w.LongWindowMetadata()) } - slices.SortFunc(windows, func(i, j model.Duration) int { - return int(int64(i) - int64(j)) + windows = append(windows, WindowMetadata{ + WindowDur: w.Period, + Name: "period", + }) + + slices.SortFunc(windows, func(i, j WindowMetadata) int { + return int(int64(i.WindowDur) - int64(j.WindowDur)) }) return windows } @@ -56,7 +60,7 @@ func (w MWMBWindows) windowRangeDur() []model.Duration { // budgeting interval is the shortest interval to monitor in a window // // budgeting interval should roughly be set to 5/432 of the SLO period -func GenerateMWMBWindows(budgetingInterval model.Duration) *MWMBWindows { +func GenerateMWMBWindows(sloPeriod, budgetingInterval model.Duration) *MWMBWindows { pageQuickShort := budgetingInterval pageQuickLong := budgetingInterval * 12 @@ -70,25 +74,34 @@ func GenerateMWMBWindows(budgetingInterval model.Duration) *MWMBWindows { ticketSlowLong := ticketQuickLong * 3 return &MWMBWindows{ + Period: sloPeriod, PageQuick: Window{ LongWindow: pageQuickLong, ShortWindow: pageQuickShort, ErrorBudgetPercent: 2, + Period: sloPeriod, + name: "page:quick", }, PageSlow: Window{ LongWindow: pageSlowLong, ShortWindow: pageSlowShort, ErrorBudgetPercent: 5, + Period: sloPeriod, + name: "page:slow", }, TicketQuick: Window{ LongWindow: ticketQuickLong, ShortWindow: ticketQuickShort, ErrorBudgetPercent: 10, + Period: sloPeriod, + name: "ticket:quick", }, TicketSlow: Window{ LongWindow: ticketSlowLong, ShortWindow: ticketSlowShort, ErrorBudgetPercent: 10, + Period: sloPeriod, + name: "ticket:slow", }, } } @@ -96,29 +109,34 @@ func GenerateMWMBWindows(budgetingInterval model.Duration) *MWMBWindows { // https://sre.google/workbook/alerting-on-slos/ func WindowDefaults(period model.Duration) *MWMBWindows { return &MWMBWindows{ + Period: period, PageQuick: Window{ ShortWindow: model.Duration(time.Minute) * 5, LongWindow: model.Duration(time.Minute) * 60, ErrorBudgetPercent: 2, Period: period, + name: "page:quick", }, PageSlow: Window{ ShortWindow: model.Duration(time.Minute) * 30, LongWindow: model.Duration(time.Hour) * 6, ErrorBudgetPercent: 5, Period: period, + name: "page:slow", }, TicketQuick: Window{ LongWindow: (model.Duration(time.Hour) * 24), ShortWindow: model.Duration(time.Hour) * 2, ErrorBudgetPercent: 10, Period: period, + name: "ticket:quick", }, TicketSlow: Window{ LongWindow: (model.Duration(time.Hour) * 24) * 3, ShortWindow: model.Duration(time.Hour) * 6, ErrorBudgetPercent: 10, Period: period, + name: "ticket:slow", }, } } @@ -138,7 +156,28 @@ type Window struct { // LongWindow is the long window used to alert based on the errors happened on that // long window. LongWindow model.Duration - Period model.Duration + // Overall SLO period + Period model.Duration + // Name of the current window, builds into prometheus rule names that depend on this window. + name string +} + +func (w Window) ShortWindowMetadata() WindowMetadata { + return WindowMetadata{ + WindowDur: w.ShortWindow, + Name: w.Name() + ":short", + } +} + +func (w Window) LongWindowMetadata() WindowMetadata { + return WindowMetadata{ + WindowDur: w.LongWindow, + Name: w.Name() + ":long", + } +} + +func (w Window) Name() string { + return w.name } func (w Window) Validate() error { @@ -182,6 +221,20 @@ func (w MWMBWindows) Validate() error { return fmt.Errorf("invalid ticket slow: %w", err) } + if w.Period < w.TicketSlow.LongWindow || + w.Period < w.TicketSlow.ShortWindow || + w.Period < w.TicketQuick.LongWindow || + w.Period < w.TicketQuick.ShortWindow { + return fmt.Errorf("period must be greater than ticket budgeting intervals") + } + + if w.Period < w.PageSlow.LongWindow || + w.Period < w.PageSlow.ShortWindow || + w.Period < w.PageQuick.LongWindow || + w.Period < w.PageQuick.ShortWindow { + return fmt.Errorf("period must be greater than page budgeting intervals") + } + return nil } diff --git a/pkg/slo/backend/metrics/window_test.go b/pkg/slo/backend/metrics/window_test.go index aa1af4efff..cc05a7246d 100644 --- a/pkg/slo/backend/metrics/window_test.go +++ b/pkg/slo/backend/metrics/window_test.go @@ -7,9 +7,10 @@ import ( . "github.com/onsi/gomega" "github.com/prometheus/common/model" "github.com/rancher/opni/pkg/slo/backend/metrics" + "github.com/samber/lo" ) -var _ = XDescribe("Windows test", Label("unit"), func() { +var _ = Describe("Windows test", Label("unit"), func() { DescribeTable("invalid Window struct", func(window metrics.Window) { Expect(window.Validate()).NotTo(Succeed()) }, @@ -30,22 +31,42 @@ var _ = XDescribe("Windows test", Label("unit"), func() { DescribeTable("valid windows -- duration strings", func(window *metrics.MWMBWindows, windowRange []string) { Expect(window.Validate()).To(Succeed()) - Expect(window.WindowRange()).To(Equal(windowRange)) + wr := window.WindowRange() + periodStrs := lo.Map(wr, func(w metrics.WindowMetadata, _ int) string { + return w.WindowDur.String() + }) + Expect(periodStrs).To(Equal(windowRange)) }, - Entry("default window", metrics.WindowDefaults(model.Duration(time.Hour*24*30)), []string{"5m", "30m", "1h", "2h", "6h", "1d", "3d"}), - Entry("custom window", metrics.GenerateMWMBWindows(model.Duration(time.Minute*5)), []string{"5m", "30m", "1h", "2h", "6h", "1d", "3d"}), - Entry("custom window short", metrics.GenerateMWMBWindows(model.Duration(time.Second*5)), []string{"5s", "30s", "1m", "2m", "6m", "24m", "1h12m"}), + Entry("default window", metrics.WindowDefaults( + model.Duration(time.Hour*24*30), + ), []string{"5m", "30m", "1h", "2h", "6h", "6h", "1d", "3d", "30d"}), + Entry("custom window", metrics.GenerateMWMBWindows( + model.Duration(time.Hour*24*28), + model.Duration(time.Minute*5), + ), []string{"5m", "30m", "1h", "2h", "6h", "6h", "1d", "3d", "4w"}), + Entry("custom window short", metrics.GenerateMWMBWindows( + model.Duration(time.Hour*5), + model.Duration(time.Second*5), + ), []string{"5s", "30s", "1m", "2m", "6m", "6m", "24m", "1h12m", "5h"}), Entry("normalizing period interval", metrics.GenerateMWMBWindows( - model.Duration(metrics.NormalizePeriodToBudgetingInterval(time.Minute*72*10))), - []string{"5s", "30s", "1m", "2m", "6m", "24m", "1h12m"}, - ), - Entry("custom window very short", metrics.GenerateMWMBWindows(model.Duration(time.Second)), []string{"1s", "6s", "12s", "24s", "1m12s", "4m48s", "14m24s"}), + model.Duration(time.Minute*72*10), + model.Duration(metrics.NormalizePeriodToBudgetingInterval(time.Minute*72*10)), + ), []string{"5s", "30s", "1m", "2m", "6m", "6m", "24m", "1h12m", "12h"}), + Entry("custom window very short", metrics.GenerateMWMBWindows( + model.Duration(time.Hour), + model.Duration(time.Second), + ), []string{"1s", "6s", "12s", "24s", "1m12s", "1m12s", "4m48s", "14m24s", "1h"}), Entry("normalizing very short interval", metrics.GenerateMWMBWindows( - model.Duration(metrics.NormalizePeriodToBudgetingInterval((time.Minute*14+(time.Second*24)))*10), - ), - []string{"1s", "6s", "12s", "24s", "1m12s", "4m48s", "14m24s"}, + model.Duration(aVeryExactDuration), + model.Duration(metrics.NormalizePeriodToBudgetingInterval(aVeryExactDuration)), + ), []string{"1s", "6s", "12s", "24s", "1m12s", "1m12s", "4m48s", "14m24s", "2h24m"}, ), ) + // TODO : burn rate factor tests }) + +var ( + aVeryExactDuration = (time.Minute*14 + (time.Second * 24)) * 10 +) diff --git a/test/plugins/metrics/slo_test.go b/test/plugins/metrics/slo_test.go index 94f02f0763..bd827f570a 100644 --- a/test/plugins/metrics/slo_test.go +++ b/test/plugins/metrics/slo_test.go @@ -174,7 +174,11 @@ var _ = Describe("Prometheus SLOs", Ordered, Label("integration"), func() { }) It("should construct loadable SLI SLI rule", func() { - expectLoadRules(env.Context(), "objective", adminClient, sloGen.SLI(prommodel.Duration(time.Second*5))) + expectLoadRules(env.Context(), "objective", adminClient, sloGen.SLI( + metrics.WindowMetadata{ + WindowDur: prommodel.Duration(time.Second * 5), + Name: "test", + })) }) It("should construct SLI current burn rate rule", func() { From a485bde9502822f88e5be6cfeb5e63051f30c3ed Mon Sep 17 00:00:00 2001 From: Alexandre Lamarre Date: Mon, 18 Sep 2023 17:22:36 -0400 Subject: [PATCH 7/7] fix lint --- test/plugins/metrics/slo_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/plugins/metrics/slo_test.go b/test/plugins/metrics/slo_test.go index bd827f570a..96d6dbfd40 100644 --- a/test/plugins/metrics/slo_test.go +++ b/test/plugins/metrics/slo_test.go @@ -612,9 +612,9 @@ func writeSLO( ts := start.Add(time.Duration(i) * time.Millisecond) cur := i % iMax if cur < iMax-iExpectedSLI { - totalVal += 1 + totalVal++ } else { - goodVal += 1 + goodVal++ } goodEvents.Samples = append(goodEvents.Samples, prompb.Sample{ Value: float64(goodVal),